HPCC-31755 Soapcall LOG multi-line separator

Signed-off-by: M Kelly <[email protected]>
hpcc-systems · Oct 8, 2024 · 56ea895 · 56ea895
1 parent 0d5a839
commit 56ea895
Show file tree

Hide file tree

Showing 7 changed files with 78 additions and 6 deletions.
diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
@@ -50,6 +50,39 @@ using roxiemem::OwnedRoxieString;
 #define CONNECTION "Connection"
 
 unsigned soapTraceLevel = 1;
+static StringBuffer soapSepString;
+static CriticalSection soapCrit;
+
+void setSoapSepString(const char *_soapSepString)
+{
+    CriticalBlock b(soapCrit);
+    soapSepString.set(_soapSepString);
+}
+
+static void multiLineAppendReplace(StringBuffer &origStr, StringBuffer &newStr)
+{
+    if (origStr.isEmpty())
+        return;
+    const char *cursor = origStr;
+    while (*cursor)
+    {
+        switch (*cursor)
+        {
+            case '\r':
+                newStr.append(soapSepString);
+                if ('\n' == *(cursor+1))
+                    cursor++;
+                break;
+            case '\n':
+                newStr.append(soapSepString);
+                break;
+            default:
+                newStr.append(*cursor);
+                break;
+        }
+        ++cursor;
+    }
+}
 
 #define WSCBUFFERSIZE 0x10000
 #define MAXWSCTHREADS 50    //Max Web Service Call Threads
@@ -1951,10 +1984,17 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
     {
         if (soapTraceLevel > 6 || master->logXML)
         {
-            if (!contentEncoded)
-                master->logctx.mCTXLOG("%s: request(%s)", master->wscCallTypeText(), request.str());
+            StringBuffer contentStr;
+            if (contentEncoded)
+                contentStr.append(", content encoded.");
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer request2;
+                multiLineAppendReplace(request, request2);
+                master->logctx.CTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request2.str(), contentStr.str());
+            }
             else
-                master->logctx.mCTXLOG("%s: request(%s), content encoded.", master->wscCallTypeText(), request.str());
+                master->logctx.mCTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request.str(), contentStr.str());
         }
     }
 
@@ -2250,9 +2290,17 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
         if (checkContentDecoding(dbgheader, response, contentEncoding))
             decodeContent(contentEncoding.str(), response);
         if (soapTraceLevel > 6 || master->logXML)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
-        else if (soapTraceLevel > 8)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but...
+        {
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer response2;
+                multiLineAppendReplace(dbgheader, response2);
+                multiLineAppendReplace(response, response2);
+                master->logctx.CTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response2.str());
+            }
+            else
+                master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
+        }
         return rval;
     }
 

diff --git a/common/thorhelper/thorsoapcall.hpp b/common/thorhelper/thorsoapcall.hpp
@@ -86,5 +86,6 @@ interface IRoxieAbortMonitor
 extern THORHELPER_API unsigned soapTraceLevel;
 extern THORHELPER_API IWSCHelper * createSoapCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
 extern THORHELPER_API IWSCHelper * createHttpCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
+extern THORHELPER_API void setSoapSepString(const char *_soapSepString);
 
 #endif /* __THORSOAPCALL_HPP_ */
diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
@@ -7738,6 +7738,10 @@ void CHThorWSCBaseActivity::init()
         JBASE64_Encode(uidpair.str(), uidpair.length(), authToken, false);
     }
     soapTraceLevel = agent.queryWorkUnit()->getDebugValueInt("soapTraceLevel", 1);
+    StringBuffer soapSepStr;
+    StringBufferAdaptor soapSepAdaptor(soapSepStr);
+    agent.queryWorkUnit()->getDebugValue("soapLogSepString", soapSepAdaptor);
+    setSoapSepString(soapSepStr.str());
 }
 
 //---------------------------------------------------------------------------

diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp
@@ -888,6 +888,12 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1);
         roxiemem::setMemTraceLevel(topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1));
         soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
+        if (topology->hasProp("@soapLogSepString"))
+        {
+            StringBuffer tmpSepString;
+            topology->getProp("@soapLogSepString", tmpSepString);
+            setSoapSepString(tmpSepString.str());
+        }
         miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0);
         traceRemoteFiles = topology->getPropBool("@traceRemoteFiles", false);
         testAgentFailure = topology->getPropInt("expert/@testAgentFailure", testAgentFailure);

diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp
@@ -2925,6 +2925,15 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme
                 soapTraceLevel = control->getPropInt("@level", 0);
                 topology->setPropInt("@soapTraceLevel", soapTraceLevel);
             }
+            else if (stricmp(queryName, "control:soapLogSepString")==0)
+            {
+                if (control->hasProp("@string"))
+                {
+                    StringBuffer tmpSepString;
+                    control->getProp("@string", tmpSepString);
+                    setSoapSepString(tmpSepString.str());
+                }
+            }
             else if (stricmp(queryName, "control:socketCheckInterval")==0)
             {
                 socketCheckInterval = (unsigned) control->getPropInt64("@val", 0);

diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp
@@ -2750,6 +2750,9 @@ void CJobBase::init()
     failOnLeaks = getOptBool(THOROPT_FAIL_ON_LEAKS);
     maxLfnBlockTimeMins = getOptInt(THOROPT_MAXLFN_BLOCKTIME_MINS, DEFAULT_MAXLFN_BLOCKTIME_MINS);
     soapTraceLevel = getOptInt(THOROPT_SOAP_TRACE_LEVEL, 1);
+    StringBuffer tmpSepString;
+    getOpt(THOROPT_SOAP_LOG_SEP_STRING, tmpSepString);
+    setSoapSepString(tmpSepString.str());
 
     StringBuffer tracing("maxActivityCores = ");
     if (maxActivityCores)

diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp
@@ -118,6 +118,7 @@
 #define THOROPT_MEMORY_SPILL_AT "memorySpillAt"                                   // The threshold (%) that roxiemem will request memory to be reduced (default=80)
 #define THOROPT_FAIL_ON_LEAKS "failOnLeaks"                                       // If any leaks are detected at the end of graph, fail the query (default=false)
 #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel"                                 // The trace SOAP level (default=1)
+#define THOROPT_SOAP_LOG_SEP_STRING "soapLogSepString"                            // The SOAP request/response separator string for logging (default="")
 #define THOROPT_SORT_ALGORITHM "sortAlgorithm"                                    // The algorithm used to sort records (quicksort/mergesort)
 #define THOROPT_COMPRESS_ALLFILES "v9_4_compressAllOutputs"                       // Compress all output files (default: bare-metal=off, cloud=on)
 #define THOROPT_AVOID_RENAME "avoidRename"                                        // Avoid rename, write directly to target physical filenames (no temp file)