diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp
index 9a246966413..94e86efdc68 100644
--- a/common/pkgfiles/referencedfilelist.cpp
+++ b/common/pkgfiles/referencedfilelist.cpp
@@ -126,13 +126,43 @@ void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringB
     prefix.append(basePrefix);
 }
 
+void splitDerivedDfsLocationOrRemote(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix,
+                                    const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix,
+                                    const char *currentRemoteStorage, StringBuffer& effectiveRemoteStorage, const char *baseRemoteStorage)
+{
+    if (!isEmptyString(address) && !isEmptyString(currentRemoteStorage))
+        throw makeStringExceptionV(-1, "Cannot specify both a dfs location (%s) and a remote storage location (%s)", address, currentRemoteStorage);
+    // Choose either a daliip (split from address) or a remote-storage location to
+    // propagate to the next level of parsing as the effective method of resolving a file
+    if (!isEmptyString(address))
+    {
+        splitDfsLocation(address, cluster, ip, prefix, defaultCluster);
+        if (!ip.isEmpty())
+            effectiveRemoteStorage.clear();
+        return;
+    }
+
+    if (!isEmptyString(currentRemoteStorage))
+    {
+        effectiveRemoteStorage.append(currentRemoteStorage);
+        // Cluster and prefix aren't used if ip is empty, so they don't need to be cleared
+        ip.clear();
+        return;
+    }
+
+    ip.append(baseIP);
+    cluster.append(baseCluster);
+    prefix.append(basePrefix);
+    effectiveRemoteStorage.append(baseRemoteStorage);
+}
+
 class ReferencedFileList;
 
 class ReferencedFile : implements IReferencedFile, public CInterface
 {
 public:
     IMPLEMENT_IINTERFACE;
-    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize)
+    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize, const char *remoteStorageName)
     : pkgid(_pkgid), fileSize(0), numParts(0), flags(_flags), noDfsResolution(noDfs), calcFileSize(calcSize), trackSubFiles(false)
     {
         {
@@ -142,6 +172,12 @@ class ReferencedFile : implements IReferencedFile, public CInterface
         }
         if (remoteStorage.length())
             flags |= RefFileLFNRemote;
+        else if (!isEmptyString(remoteStorageName))
+        {
+            // can be declared in packagemap at different scopes
+            remoteStorage.set(remoteStorageName);
+            flags |= RefFileLFNRemote;
+        }
         else if (daliip.length())
             flags |= RefFileLFNForeign;
         else
@@ -152,6 +188,9 @@ class ReferencedFile : implements IReferencedFile, public CInterface
             flags |= RefSubFile;
     }
 
+    ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize)
+    : ReferencedFile(lfn, sourceIP, srcCluster, prefix, isSubFile, _flags, _pkgid, noDfs, calcSize, nullptr) {    }
+
     void reset()
     {
         flags &= ~(RefFileNotOnCluster | RefFileNotFound | RefFileResolvedForeign | RefFileResolvedRemote | RefFileCopyInfoFailed | RefFileCloned | RefFileNotOnSource); //these flags are calculated during resolve
@@ -242,18 +281,18 @@ class ReferencedFileList : implements IReferencedFileList, public CInterface
             user.set(userDesc);
     }
 
-    void ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
+    void ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip=nullptr, const char *srcCluster=nullptr, const char *remotePrefix=nullptr, const char* remoteStorage=nullptr);
 
-    virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL);
+    virtual void addFile(const char *ln, const char *daliip=nullptr, const char *srcCluster=nullptr, const char *remotePrefix=nullptr, const char *remoteStorageName=nullptr);
     virtual void addFiles(StringArray &files);
     virtual void addFilesFromWorkUnit(IConstWorkUnit *cw);
     virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid);
     virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg);
     virtual void addFilesFromPackageMap(IPropertyTree *pm);
 
-    void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
-    void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
-    void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster, const char *_remotePrefix);
+    void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName);
+    void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName);
+    void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName);
 
     virtual IReferencedFileIterator *getFiles();
     virtual void cloneFileInfo(StringBuffer &publisherWuid, const char *dstCluster, unsigned updateFlags, IDFUhelper *helper, bool cloneSuperInfo, bool cloneForeign, unsigned redundancy, unsigned channelsPerNode, int replicateOffset, const char *defReplicateFolder);
@@ -883,12 +922,12 @@ class ReferencedFileIterator : implements IReferencedFileIterator, public CInter
     Owned<HashIterator> iter;
 };
 
-void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip, const char *srcCluster, const char *prefix)
+void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip, const char *srcCluster, const char *prefix, const char *remoteStorageName)
 {
     if (!allowForeign && checkForeign(ln))
         throw MakeStringException(-1, "Foreign file not allowed%s: %s", (flags & RefFileInPackage) ? " (declared in package)" : "", ln);
 
-    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution, allowSizeCalc);
+    Owned<ReferencedFile> file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution, allowSizeCalc, remoteStorageName);
     if (!file->logicalName.length())
         return;
     if (subfileNames)
@@ -904,9 +943,9 @@ void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *
     }
 }
 
-void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster, const char *prefix)
+void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster, const char *prefix, const char *remoteStorageName)
 {
-    ensureFile(ln, 0, NULL, false, nullptr, daliip, srcCluster, prefix);
+    ensureFile(ln, 0, NULL, false, nullptr, daliip, srcCluster, prefix, remoteStorageName);
 }
 
 void ReferencedFileList::addFiles(StringArray &files)
@@ -915,37 +954,41 @@ void ReferencedFileList::addFiles(StringArray &files)
         addFile(files.item(i));
 }
 
-void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *ip, const char *cluster, const char *prefix)
+void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *ip, const char *cluster, const char *prefix, const char *remoteStorageName)
 {
-    addFile(subFile.queryProp("@value"), ip, cluster, prefix);
+    addFile(subFile.queryProp("@value"), ip, cluster, prefix, remoteStorageName);
 }
 
-void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *_ip, const char *_cluster, const char *_prefix)
+void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *_ip, const char *_cluster, const char *_prefix, const char *ancestorRemoteStorage)
 {
     StringBuffer ip;
     StringBuffer cluster;
     StringBuffer prefix;
-    splitDerivedDfsLocation(superFile.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix);
+    StringBuffer effectiveRemoteStorage;
+    splitDerivedDfsLocationOrRemote(superFile.queryProp("@daliip"), cluster, ip, prefix, nullptr, _cluster, _ip, _prefix,
+                                    superFile.queryProp("@remoteStorage"), effectiveRemoteStorage, ancestorRemoteStorage);
     if (superFile.hasProp("@sourceCluster"))
         cluster.set(superFile.queryProp("@sourceCluster"));
 
     Owned<IPropertyTreeIterator> subFiles = superFile.getElements("SubFile[@value]");
     ForEach(*subFiles)
-        addFileFromSubFile(subFiles->query(), ip, cluster, prefix);
+        addFileFromSubFile(subFiles->query(), ip, cluster, prefix, effectiveRemoteStorage);
 }
 
-void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *_ip, const char *_cluster, const char *_prefix)
+void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *_ip, const char *_cluster, const char *_prefix, const char *ancestorRemoteStorage)
 {
     StringBuffer ip;
     StringBuffer cluster;
     StringBuffer prefix;
-    splitDerivedDfsLocation(package.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix);
+    StringBuffer effectiveRemoteStorage;
+    splitDerivedDfsLocationOrRemote(package.queryProp("@daliip"), cluster, ip, prefix, nullptr, _cluster, _ip, _prefix,
+                                    package.queryProp("@remoteStorage"), effectiveRemoteStorage, ancestorRemoteStorage);
     if (package.hasProp("@sourceCluster"))
         cluster.set(package.queryProp("@sourceCluster"));
 
     Owned<IPropertyTreeIterator> supers = package.getElements("SuperFile");
     ForEach(*supers)
-        addFilesFromSuperFile(supers->query(), ip, cluster, prefix);
+        addFilesFromSuperFile(supers->query(), ip, cluster, prefix, effectiveRemoteStorage);
 }
 
 void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)
@@ -953,20 +996,22 @@ void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm)
     StringBuffer ip;
     StringBuffer cluster;
     StringBuffer prefix;
+    StringBuffer remoteStorageName;
     const char *srcCluster = pm->queryProp("@sourceCluster");
-    splitDerivedDfsLocation(pm->queryProp("@daliip"), cluster, ip, prefix, srcCluster, srcCluster, NULL, NULL);
+    splitDerivedDfsLocationOrRemote(pm->queryProp("@daliip"), cluster, ip, prefix, srcCluster, srcCluster, nullptr, nullptr,
+                                    pm->queryProp("@remoteStorage"), remoteStorageName, nullptr);
     Owned<IPropertyTreeIterator> packages = pm->getElements("Package");
     ForEach(*packages)
-        addFilesFromPackage(packages->query(), ip, cluster, prefix);
+        addFilesFromPackage(packages->query(), ip, cluster, prefix, remoteStorageName);
     packages.setown(pm->getElements("Part/Package"));
     ForEach(*packages)
-        addFilesFromPackage(packages->query(), ip, cluster, prefix);
+        addFilesFromPackage(packages->query(), ip, cluster, prefix, remoteStorageName);
 }
 
 bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)
 {
     SummaryMap files;
-    if (cw->getSummary(SummaryType::ReadFile, files) && 
+    if (cw->getSummary(SummaryType::ReadFile, files) &&
         cw->getSummary(SummaryType::ReadIndex, files))
     {
         for (const auto& [lName, summaryFlags] : files)
@@ -1104,31 +1149,43 @@ void ReferencedFileList::resolveFiles(const StringArray &locations, const char *
     srcCluster.set(_srcCluster);
     remotePrefix.set(_remotePrefix);
 
+    ReferencedFileIterator files(this);
+
+    // For use when expandSuperFiles=true
     if (useRemoteStorage)
-    {
-        DBGLOG("ReferencedFileList resolving remote storage files at %s", nullText(remoteLocation));
-        if (!user)
-            user.setown(createUserDescriptor());
         remoteStorage.set(remoteLocation);
-        ReferencedFileIterator files(this);
-        ForEach(files)
-            files.queryObject().resolveLocalOrRemote(locations, srcCluster, user, remoteStorage, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign);
-    }
-    else
+
+    ForEach(files)
     {
-        if (!isEmptyString(remoteLocation))
-            DBGLOG("ReferencedFileList resolving remote dali files at %s", remoteLocation);
+        ReferencedFile &file = files.queryObject();
+        if (file.daliip.isEmpty() && (!file.remoteStorage.isEmpty() || useRemoteStorage))
+        {
+            if (!user)
+                user.setown(createUserDescriptor());
+
+            if (file.remoteStorage.isEmpty()) // Can be set at multiple levels in a packagemap
+                file.remoteStorage.set(remoteLocation); // Top-level remoteLocation has lowest precedence, used if nothing set in packagemap
+
+            DBGLOG("ReferencedFileList resolving remote storage file at %s", nullText(file.remoteStorage));
+            file.resolveLocalOrRemote(locations, srcCluster, user, file.remoteStorage, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign);
+        }
         else
-            DBGLOG("ReferencedFileList resolving local files (no daliip)");
-        remote.setown(!isEmptyString(remoteLocation) ? createINode(remoteLocation, 7070) : nullptr);
+        {
+            // The remoteLocation is a daliip when useRemoteStorage is false
+            const char *passedDaliip = !useRemoteStorage ? remoteLocation : nullptr;
+            if (!isEmptyString(passedDaliip) || !file.daliip.isEmpty())
+                DBGLOG("ReferencedFileList resolving remote dali file at %s", isEmptyString(passedDaliip) ? nullText(file.daliip) : passedDaliip);
+            else
+                DBGLOG("ReferencedFileList resolving local file (no daliip or remote storage)");
+            // Otherwise, passing nullptr for remote allows resolveLocalOrForeign to use ReferencedFile.daliip with
+            // the matching ReferencedFile.remotePrefix instead of the ReferencedFileList.remotePrefix passed in here.
+            remote.setown(!isEmptyString(passedDaliip) ? createINode(passedDaliip, 7070) : nullptr);
+            file.resolveLocalOrForeign(locations, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign);
+        }
 
-        ReferencedFileIterator files(this);
-        ForEach(files)
-            files.queryObject().resolveLocalOrForeign(locations, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign);
+        if (expandSuperFiles)
+            resolveSubFiles(subfiles, locations, checkLocalFirst, trackSubFiles, resolveLFNForeign);
     }
-
-    if (expandSuperFiles)
-        resolveSubFiles(subfiles, locations, checkLocalFirst, trackSubFiles, resolveLFNForeign);
 }
 
 bool ReferencedFileList::filesNeedCopying(bool cloneForeign)
diff --git a/common/pkgfiles/referencedfilelist.hpp b/common/pkgfiles/referencedfilelist.hpp
index 3d20b40f7c3..41421b5a812 100644
--- a/common/pkgfiles/referencedfilelist.hpp
+++ b/common/pkgfiles/referencedfilelist.hpp
@@ -68,7 +68,7 @@ interface IReferencedFileList : extends IInterface
     virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0;
     virtual void addFilesFromPackageMap(IPropertyTree *pm)=0;
 
-    virtual void addFile(const char *ln, const char *daliip=NULL, const char *sourceProcessCluster=NULL, const char *remotePrefix=NULL)=0;
+    virtual void addFile(const char *ln, const char *daliip=nullptr, const char *sourceProcessCluster=nullptr, const char *remotePrefix=nullptr, const char *remoteStorageName=nullptr)=0;
     virtual void addFiles(StringArray &files)=0;
 
     virtual IReferencedFileIterator *getFiles()=0;
diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp
index c7e3df41b05..c696e5e8624 100644
--- a/common/thorhelper/thorsoapcall.cpp
+++ b/common/thorhelper/thorsoapcall.cpp
@@ -50,6 +50,40 @@ using roxiemem::OwnedRoxieString;
 #define CONNECTION "Connection"
 
 unsigned soapTraceLevel = 1;
+static StringBuffer soapSepString;
+
+void setSoapSepString(const char *_soapSepString)
+{
+    soapSepString.set(_soapSepString);
+}
+
+static void multiLineAppendReplace(StringBuffer &origStr, StringBuffer &newStr)
+{
+    if (origStr.isEmpty())
+        return;
+
+    newStr.ensureCapacity(origStr.length());
+
+    const char *cursor = origStr;
+    while (*cursor)
+    {
+        switch (*cursor)
+        {
+            case '\r':
+                newStr.append(soapSepString);
+                if ('\n' == *(cursor+1))
+                    cursor++;
+                break;
+            case '\n':
+                newStr.append(soapSepString);
+                break;
+            default:
+                newStr.append(*cursor);
+                break;
+        }
+        ++cursor;
+    }
+}
 
 #define WSCBUFFERSIZE 0x10000
 #define MAXWSCTHREADS 50    //Max Web Service Call Threads
@@ -1940,10 +1974,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
     {
         if (soapTraceLevel > 6 || master->logXML)
         {
-            if (!contentEncoded)
-                master->logctx.mCTXLOG("%s: request(%s)", master->wscCallTypeText(), request.str());
+            StringBuffer contentStr;
+            if (contentEncoded)
+                contentStr.append(", content encoded.");
+            // Only do translation if soapcall LOG option set and soapSepString defined
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer request2;
+                multiLineAppendReplace(request, request2);
+                master->logctx.CTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request2.str(), contentStr.str());
+            }
             else
-                master->logctx.mCTXLOG("%s: request(%s), content encoded.", master->wscCallTypeText(), request.str());
+                master->logctx.mCTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request.str(), contentStr.str());
         }
     }
 
@@ -2247,9 +2289,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo
         if (checkContentDecoding(dbgheader, response, contentEncoding))
             decodeContent(contentEncoding.str(), response);
         if (soapTraceLevel > 6 || master->logXML)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
-        else if (soapTraceLevel > 8)
-            master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but...
+        {
+            // Only do translation if soapcall LOG option set and soapSepString defined
+            if ( (master->logXML) && (soapSepString.length() > 0) )
+            {
+                StringBuffer response2;
+                multiLineAppendReplace(dbgheader, response2);
+                multiLineAppendReplace(response, response2);
+                master->logctx.CTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response2.str());
+            }
+            else
+                master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str());
+        }
         return rval;
     }
 
diff --git a/common/thorhelper/thorsoapcall.hpp b/common/thorhelper/thorsoapcall.hpp
index aa1c97a6c22..4ea42feef69 100644
--- a/common/thorhelper/thorsoapcall.hpp
+++ b/common/thorhelper/thorsoapcall.hpp
@@ -86,5 +86,6 @@ interface IRoxieAbortMonitor
 extern THORHELPER_API unsigned soapTraceLevel;
 extern THORHELPER_API IWSCHelper * createSoapCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
 extern THORHELPER_API IWSCHelper * createHttpCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor);
+extern THORHELPER_API void setSoapSepString(const char *_soapSepString);
 
 #endif /* __THORSOAPCALL_HPP_ */
diff --git a/dali/base/CMakeLists.txt b/dali/base/CMakeLists.txt
index 7046aae6bcb..07e7b44695a 100644
--- a/dali/base/CMakeLists.txt
+++ b/dali/base/CMakeLists.txt
@@ -39,7 +39,8 @@ set (    SRCS
          dasds.cpp 
          dasess.cpp 
          dasubs.cpp 
-         dautils.cpp 
+         dautils.cpp
+         sysinfologger.cpp
     )
     
 set (    INCLUDES
@@ -56,6 +57,7 @@ set (    INCLUDES
          dasess.hpp
          dasubs.hpp
          dautils.hpp
+         sysinfologger.hpp
     )   
 
 set_source_files_properties(dasds.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
@@ -70,6 +72,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/rtl/include 
          ${HPCC_SOURCE_DIR}/system/security/shared
          ${HPCC_SOURCE_DIR}/system/security/cryptohelper
+         ${HPCC_SOURCE_DIR}/testing/unittests
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DDALI_EXPORTS -DNULL_DALIUSER_STACKTRACE)
@@ -88,5 +91,6 @@ if(NOT PLUGIN)
         mp 
         hrpc 
         dafsclient
+        ${CppUnit_LIBRARIES}
     )
 endif()
diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp
new file mode 100644
index 00000000000..242d303733f
--- /dev/null
+++ b/dali/base/sysinfologger.cpp
@@ -0,0 +1,696 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "sysinfologger.hpp"
+#include "daclient.hpp"
+#include "jutil.hpp"
+
+#define SDS_LOCK_TIMEOUT (5*60*1000) // 5 minutes
+#define SYS_INFO_VERSION "1.0"
+
+#define SYS_INFO_ROOT  "/SysLogs"
+#define ATTR_NEXTID    "@nextId"
+#define ATTR_VERSION   "@version"
+#define MSG_NODE       "msg"
+#define ATTR_ID        "@id"
+#define ATTR_TIMESTAMP "@ts"
+#define ATTR_AUDIENCE  "@audience"
+#define ATTR_CLASS     "@class"
+#define ATTR_CODE      "@code"
+#define ATTR_HIDDEN    "@hidden"
+#define ATTR_SOURCE    "@source"
+
+static void extractDate(timestamp_type ts, unsigned & year, unsigned & month, unsigned & day)
+{
+    CDateTime timeStamp;
+    timeStamp.setTimeStamp(ts);
+    timeStamp.getDate(year, month, day);
+}
+
+static unsigned __int64 makeMessageId(unsigned year, unsigned month, unsigned day, unsigned id)
+{
+    return id<<21 | year<<9 | month<<5 | day;
+}
+
+static unsigned __int64 makeMessageId(unsigned __int64 ts, unsigned id)
+{
+    unsigned year, month, day;
+    extractDate(ts, year, month, day);
+    return makeMessageId(year, month, day, id);
+}
+
+static void decodeMessageId(unsigned __int64 msgId, unsigned & year, unsigned & month, unsigned & day, unsigned & id)
+{
+    day = msgId & 0x1F;
+    month = (msgId>>5) & 0x0F;
+    year = (msgId>>9) & 0xFFF;
+    id = (msgId>>21);
+}
+
+class CSysInfoLoggerMsg : implements ISysInfoLoggerMsg
+{
+    Owned<IPropertyTree> msgPtree;
+    bool updateable = false;
+
+    inline void ensureUpdateable()
+    {
+        if (!updateable)
+            throw makeStringException(-1, "Unable to update ISysInfoLoggerMsg");
+    }
+
+public:
+    CSysInfoLoggerMsg()
+    {
+        msgPtree.setown(createPTree(MSG_NODE));
+    }
+    CSysInfoLoggerMsg(unsigned id, const LogMsgCategory & cat, LogMsgCode code, const char * source, const char * msg, timestamp_type ts, bool hidden)
+    {
+        msgPtree.setown(createPTree(MSG_NODE));
+        msgPtree->setPropInt64(ATTR_ID, id);
+        msgPtree->setPropBool(ATTR_HIDDEN, hidden);
+        msgPtree->setPropInt64(ATTR_TIMESTAMP, ts);
+        msgPtree->setPropInt(ATTR_CODE, code);
+        msgPtree->setProp(ATTR_AUDIENCE, LogMsgAudienceToFixString(cat.queryAudience()));
+        msgPtree->setProp(ATTR_CLASS, LogMsgClassToFixString(cat.queryClass()));
+        msgPtree->setProp(ATTR_SOURCE, source);
+        msgPtree->setProp(".", msg);
+    }
+    CSysInfoLoggerMsg & set(IPropertyTree * ptree, bool _updateable)
+    {
+        msgPtree.setown(ptree);
+        updateable = _updateable;
+        return * this;
+    }
+    virtual bool queryIsHidden() const override
+    {
+        return msgPtree->getPropBool(ATTR_HIDDEN, false);
+    }
+    virtual timestamp_type queryTimeStamp() const override
+    {
+        return msgPtree->getPropInt64(ATTR_TIMESTAMP);
+    }
+    virtual const char * querySource() const override
+    {
+        if (msgPtree->hasProp(ATTR_SOURCE))
+            return msgPtree->queryProp(ATTR_SOURCE);
+        else
+            return "Unknown";
+    }
+    virtual LogMsgCode queryLogMsgCode() const override
+    {
+        return msgPtree->getPropInt(ATTR_CODE, -1);
+    }
+    virtual LogMsgAudience queryAudience() const override
+    {
+        if (msgPtree->hasProp(ATTR_AUDIENCE))
+            return LogMsgAudFromAbbrev(msgPtree->queryProp(ATTR_AUDIENCE));
+        else
+            return MSGAUD_unknown;
+    }
+    virtual LogMsgClass queryClass() const override
+    {
+        if (msgPtree->hasProp(ATTR_CLASS))
+            return LogMsgClassFromAbbrev(msgPtree->queryProp(ATTR_CLASS));
+        else
+            return MSGCLS_unknown;
+    }
+    virtual unsigned __int64 queryLogMsgId() const override
+    {
+        return makeMessageId(queryTimeStamp(), msgPtree->getPropInt64(ATTR_ID, 0));
+    }
+    virtual const char * queryMsg() const override
+    {
+        const char *msg = msgPtree->queryProp(nullptr);
+        return msg ? msg : "";
+    }
+    void setHidden(bool _hidden)
+    {
+        ensureUpdateable();
+        msgPtree->setPropBool(ATTR_HIDDEN, _hidden);
+    }
+    StringBuffer & getXpath(StringBuffer & xpath)
+    {
+        unsigned year, month, day;
+        extractDate(queryTimeStamp(), year, month, day);
+        unsigned __int64 id = msgPtree->getPropInt64(ATTR_ID, 0);
+        xpath.appendf("m%04u%02u/d%02u/" MSG_NODE "[" ATTR_ID "='%" I64F "u']", year, month, day, id);
+        return xpath;
+    }
+    IPropertyTree * getTree()
+    {
+        return msgPtree.getLink();
+    }
+};
+
+class CSysInfoLoggerMsgFilter : public CSimpleInterfaceOf<ISysInfoLoggerMsgFilter>
+{
+    // (For numeric fields: match only if it has a non-zero value)
+    bool hiddenOnly = false;
+    bool visibleOnly = false;
+    timestamp_type matchTimeStamp = 0;
+    StringAttr matchSource; // only matchSource when not empty
+    LogMsgCode matchCode = 0;
+    LogMsgAudience matchAudience = MSGAUD_all;
+    LogMsgClass matchClass = MSGCLS_all;
+    bool haveDateRange = false;
+    unsigned matchEndYear = 0;
+    unsigned matchEndMonth = 0;
+    unsigned matchEndDay = 0;
+    unsigned matchStartYear = 0;
+    unsigned matchStartMonth = 0;
+    unsigned matchStartDay = 0;
+    unsigned matchId = 0;
+
+public:
+    CSysInfoLoggerMsgFilter(const char *_source): matchSource(_source)
+    {
+    }
+    CSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *_source): matchSource(_source)
+    {
+        setMatchMsgId(msgId);
+    }
+    CSysInfoLoggerMsgFilter(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *_source) :
+                            visibleOnly(_visibleOnly), hiddenOnly(_hiddenOnly), matchSource(_source)
+    {
+        if (hiddenOnly && visibleOnly)
+            throw makeStringException(-1, "ISysInfoLoggerMsgFilter: cannot filter by both hiddenOnly and visibleOnly");
+        setDateRange(_year, _month, _day, _year, _month, _day);
+    }
+    virtual void setHiddenOnly() override
+    {
+        hiddenOnly = true;
+    }
+    virtual void setVisibleOnly() override
+    {
+        visibleOnly = true;
+    }
+    virtual void setMatchTimeStamp(timestamp_type ts) override
+    {
+        matchTimeStamp = ts;
+    }
+    virtual void setMatchSource(const char * source) override
+    {
+        matchSource.set(source);
+    }
+    virtual void setMatchCode(LogMsgCode code) override
+    {
+        matchCode = code;
+    }
+    virtual void setMatchAudience(LogMsgAudience audience) override
+    {
+        matchAudience = audience;
+    }
+    virtual void setMatchMsgClass(LogMsgClass msgClass) override
+    {
+        matchClass = msgClass;
+    }
+    virtual void setMatchMsgId(unsigned __int64 msgId) override
+    {
+        unsigned year, month, day, id;
+        decodeMessageId(msgId, year, month, day, id);
+        if (year==0 || month==0 || day==0 || id==0)
+            throw makeStringExceptionV(-1,"ISysInfoLoggerMsgFilter::setMatchMsgId invalid argument: %" I64F "u", msgId);
+        matchEndYear = matchStartYear = year;
+        matchEndMonth = matchStartMonth = month;
+        matchEndDay = matchStartDay = day;
+        matchId = id;
+        haveDateRange = false;
+    }
+    virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) override
+    {
+        if ( (startDay && (!startMonth||!startYear)) ||
+             (endDay && (!endMonth||!endYear)) )
+            throw makeStringException(-1, "ISysInfoLoggerMsgFilter: month and year must be provided when filtering by day");
+        if ((!startYear && startMonth) || (!endYear && endMonth))
+            throw makeStringException(-1, "ISysInfoLoggerMsgFilter: year must be provided when filtering by month");
+        // Make sure starts are on or before end dates
+        if ( (startYear > endYear) || (startMonth && (startYear == endYear && startMonth > endMonth))
+             || (startDay && (startYear == endYear && startMonth == endMonth && startDay > endDay)) )
+            throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: invalid date range: %04u-%02u-%02u to %04u-%02u-%02u", startYear, startMonth, startDay, endYear, endMonth, endDay);
+        matchEndYear = endYear;
+        matchEndMonth = endMonth;
+        matchEndDay = endDay;
+        matchStartYear = startYear;
+        matchStartMonth = startMonth;
+        matchStartDay = startDay;
+        if (matchEndYear||matchStartYear)
+            haveDateRange = (matchStartYear<matchEndYear)||(matchStartMonth<matchEndMonth)||(matchStartDay<matchEndDay);
+        else
+            haveDateRange = false;
+    }
+    virtual void setOlderThanDate(unsigned year, unsigned month, unsigned day) override
+    {
+        assert(year);
+        matchEndYear = year;
+        matchEndMonth = month;
+        matchEndDay = day;
+        matchStartYear = 0;
+        matchStartMonth = 0;
+        matchStartDay = 0;
+        haveDateRange = true;
+    }
+    virtual bool hasDateRange() const override
+    {
+        return haveDateRange;
+    }
+    virtual bool isInDateRange(timestamp_type ts) const override
+    {
+        unsigned tyear, tmonth, tday;
+        extractDate(ts, tyear, tmonth, tday);
+
+        if (matchStartYear)
+        {
+            if (tyear<matchStartYear)
+                return false;
+            if (tyear==matchStartYear)
+            {
+                if (matchStartMonth)
+                {
+                    if (tmonth<matchStartMonth)
+                        return false;
+                    if (tmonth==matchStartMonth)
+                    {
+                        if (matchStartDay && tday<matchStartDay)
+                            return false;
+                    }
+                }
+            }
+        }
+        if (matchEndYear)
+        {
+            if (tyear>matchEndYear)
+                return false;
+            if (tyear==matchEndYear)
+            {
+                if (matchEndMonth)
+                {
+                    if (tmonth>matchEndMonth)
+                        return false;
+                    if (tmonth==matchEndMonth)
+                    {
+                        if (matchEndDay && tday>matchEndDay)
+                            return false;
+                    }
+                }
+            }
+        }
+        return true;
+    }
+    virtual bool queryHiddenOnly() const override
+    {
+        return hiddenOnly;
+    }
+    virtual bool queryVisibleOnly() const override
+    {
+        return visibleOnly;
+    }
+    virtual timestamp_type queryMatchTimeStamp() const override
+    {
+        return matchTimeStamp;
+    }
+    virtual unsigned queryStartYear() const override
+    {
+        return matchStartYear;
+    }
+    virtual unsigned queryStartMonth() const override
+    {
+        return matchStartMonth;
+    }
+    virtual unsigned queryStartDay() const override
+    {
+        return matchStartDay;
+    }
+    virtual unsigned queryEndYear() const override
+    {
+        return matchEndYear;
+    }
+    virtual unsigned queryEndMonth() const override
+    {
+        return matchEndMonth;
+    }
+    virtual unsigned queryEndDay() const override
+    {
+        return matchEndDay;
+    }
+    virtual const char * queryMatchSource() const override
+    {
+        return matchSource.str();
+    }
+    virtual LogMsgCode queryMatchCode() const override
+    {
+        return matchCode;
+    }
+    virtual LogMsgAudience queryMatchAudience() const override
+    {
+        return matchAudience;
+    }
+    virtual LogMsgClass queryMatchClass() const override
+    {
+        return matchClass;
+    }
+    virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const override
+    {
+        bool fullDayMatch=false;
+        bool hardMatchYear = matchStartYear && (matchStartYear==matchEndYear);
+        bool hardMatchMonth = matchStartMonth && (matchStartMonth==matchEndMonth);
+        if (hardMatchYear && hardMatchMonth)
+        {
+            // future: optimize when month unknown with "m%04u*"
+            xpath.appendf("m%04u%02u", matchStartYear, matchStartMonth);
+            if (matchStartDay==matchEndDay)
+            {
+                xpath.appendf("/d%02u", matchStartDay);
+                fullDayMatch = true;
+            }
+        }
+        if (fullDayMatch)
+            xpath.appendf("/" MSG_NODE);
+        else
+            xpath.appendf("//" MSG_NODE);
+        if (hiddenOnly)
+            xpath.append("[" ATTR_HIDDEN "='1')]");
+        if (visibleOnly)
+            xpath.append("[" ATTR_HIDDEN "='0')]");
+        if (!matchSource.isEmpty())
+            xpath.appendf("[" ATTR_SOURCE "='%s']", matchSource.str());
+        if (matchCode)
+            xpath.appendf("[" ATTR_CODE "='%d']", (int)matchCode);
+        if (matchAudience!=MSGAUD_all)
+            xpath.appendf("[" ATTR_AUDIENCE "='%s']", LogMsgAudienceToFixString(matchAudience));
+        if (matchClass!=MSGCLS_all)
+            xpath.appendf("[" ATTR_CLASS "='%s']", LogMsgClassToFixString(matchClass));
+        if (matchId)
+            xpath.appendf("[" ATTR_ID "='%u']", matchId);
+        if (matchTimeStamp)
+            xpath.appendf("[" ATTR_TIMESTAMP "='%" I64F "u']", matchTimeStamp);
+        return xpath;
+    }
+};
+
+ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source)
+{
+    return new CSysInfoLoggerMsgFilter(source);
+}
+
+ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source)
+{
+    return new CSysInfoLoggerMsgFilter(msgId, source);
+}
+
+class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOf<ISysInfoLoggerMsgIterator>
+{
+    Linked<IConstSysInfoLoggerMsgFilter> filter;
+    // N.b. IRemoteConnection exists for the duration of the iterator so if this iterator exists for too long, it could cause
+    // performance issues for other clients:  consider caching some messages and releasing connection (and reopening as necessary).
+    Owned<IRemoteConnection> conn;
+    bool updateable = false;
+    Owned<IPropertyTreeIterator> msgIter;
+    CSysInfoLoggerMsg infoMsg;
+
+    bool ensureMatch()
+    {
+        if (filter->hasDateRange())
+        {
+            for (; msgIter->isValid(); msgIter->next())
+            {
+                timestamp_type ts = msgIter->query().getPropInt64(ATTR_TIMESTAMP, 0);
+                if (filter->isInDateRange(ts))
+                    return true;
+            }
+            return false;
+        }
+        return msgIter->isValid();
+    }
+
+public:
+    CSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * _filter, bool _updateable=false, IRemoteConnection *_conn=nullptr) : filter(_filter), updateable(_updateable), conn(_conn)
+    {
+        if (!conn)
+        {
+            unsigned mode = updateable ? RTM_LOCK_WRITE : RTM_LOCK_READ;
+            conn.setown(querySDS().connect(SYS_INFO_ROOT, myProcessSession(), mode, SDS_LOCK_TIMEOUT));
+            if (!conn)
+                throw makeStringExceptionV(-1, "CSysInfoLoggerMsgIterator: unable to create connection to '%s'", SYS_INFO_ROOT);
+        }
+    }
+    CSysInfoLoggerMsg & queryInfoLoggerMsg()
+    {
+        return infoMsg.set(&(msgIter->get()), updateable);
+    }
+    virtual ISysInfoLoggerMsg & query() override
+    {
+        return queryInfoLoggerMsg();
+    }
+    virtual bool first() override
+    {
+        StringBuffer xpath;
+        filter->getQualifierXPathFilter(xpath);
+
+        msgIter.setown(conn->queryRoot()->getElements(xpath.str()));
+        if (!msgIter->first())
+            return false;
+        return ensureMatch();
+    }
+    virtual bool next() override
+    {
+        if (msgIter->next())
+            return false;
+        return ensureMatch();
+    }
+    virtual bool isValid() override
+    {
+        return msgIter ? msgIter->isValid() : false;
+    }
+};
+
+ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source)
+{
+    Owned<CSysInfoLoggerMsgFilter> filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day, source);
+    return new CSysInfoLoggerMsgIterator(filter, false);
+}
+
+ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter)
+{
+    return new CSysInfoLoggerMsgIterator(msgFilter, false);
+}
+
+// returns messageId
+unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, timestamp_type ts)
+{
+    if (ts==0)
+        ts = getTimeStampNowValue();
+
+    if (isEmptyString(source))
+        source = "unknown";
+    Owned<IRemoteConnection> conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", SYS_INFO_ROOT);
+
+    IPropertyTree * root = conn->queryRoot();
+    unsigned id = root->getPropInt(ATTR_NEXTID, 1);
+    if (id==UINT_MAX)  // wrap id to reuse id numbers (shouldn't wrap but no harm in doing this for safety)
+        id=1;
+    root->setPropInt(ATTR_NEXTID, id+1);
+
+    StringBuffer xpath;
+    unsigned year, month, day;
+    extractDate(ts, year, month, day);
+    xpath.appendf("%s/m%04u%02u/d%02u/%s", SYS_INFO_ROOT, year, month, day, MSG_NODE);
+    Owned<IRemoteConnection> connMsgRoot = querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE_ADD, SDS_LOCK_TIMEOUT);
+    if (!connMsgRoot)
+        throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", xpath.str());
+    IPropertyTree * msgPT = connMsgRoot->queryRoot();
+
+    CSysInfoLoggerMsg sysInfoMsg(id, cat, code, source, msg, ts, false);
+    msgPT->setPropTree(nullptr, sysInfoMsg.getTree());
+    msgPT->setProp(".", msg); // previous setPropTree doesn't set the node value
+    return makeMessageId(ts, id);
+}
+
+unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function<void (CSysInfoLoggerMsg &)> updateOp)
+{
+    unsigned count = 0;
+    Owned<CSysInfoLoggerMsgIterator> iter = new CSysInfoLoggerMsgIterator(msgFilter, true);
+    ForEach(*iter)
+    {
+        CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg();
+        updateOp(sysInfoMsg);
+        ++count;
+    }
+    return count;
+}
+
+unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function<void (CSysInfoLoggerMsg &)> updateOp)
+{
+    Owned<IConstSysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter(msgId, source);
+    return updateMessage(msgFilter, updateOp);
+}
+
+unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter)
+{
+    return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);});
+}
+
+bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source)
+{
+    return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1;
+}
+
+unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter)
+{
+    return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);});
+}
+
+bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source)
+{
+    return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1;
+}
+
+unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter)
+{
+    std::vector<std::string> deleteXpathList;
+    Owned<IRemoteConnection> conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    {
+        Owned<CSysInfoLoggerMsgIterator> iter = new CSysInfoLoggerMsgIterator(msgFilter, false, conn.getLink());
+        ForEach(*iter)
+        {
+            CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg();
+            StringBuffer xpath;
+            sysInfoMsg.getXpath(xpath);
+            deleteXpathList.push_back(xpath.str());
+        }
+    }
+    IPropertyTree * root = conn->queryRoot();
+    unsigned count = 0;
+    for (auto & xpath: deleteXpathList)
+    {
+        if (root->removeProp(xpath.c_str()))
+            ++count;
+    }
+    return count;
+}
+
+bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source)
+{
+    Owned<ISysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter(msgId, source);
+    return deleteLogSysInfoMsg(msgFilter);
+}
+
+unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source)
+{
+    if (!year && month)
+        throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: year must be provided if month is specified (year=%u, month=%u, day=%u)", year, month, day);
+    if (!month && day)
+        throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: month must be provided if day is specified (year=%u, month=%u, day=%u)", year, month, day);
+    if (month>12)
+        throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid month(year=%u, month=%u, day=%u)", year, month, day);
+    if (day>31)
+        throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid day(year=%u, month=%u, day=%u)", year, month, day);
+    // With visibleOnly/hiddenOnly option, use createSysInfoLoggerMsgFilter()
+    if (visibleOnly || hiddenOnly || day)
+    {
+        unsigned count = 0;
+        Owned<ISysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter(source);
+        if (hiddenOnly)
+            msgFilter->setHiddenOnly();
+        if (visibleOnly)
+            msgFilter->setVisibleOnly();
+        if (source)
+            msgFilter->setMatchSource(source);
+        msgFilter->setOlderThanDate(year, month, day);
+        return deleteLogSysInfoMsg(msgFilter);
+    }
+
+    // With only date range, use this quicker method to remove whole subtrees
+    Owned<IRemoteConnection> conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT);
+    if (!conn)
+        return 0;
+
+    std::vector<std::string> deleteXpathList;
+    IPropertyTree * root = conn->queryRoot();
+    // future: optimize by getting only minimum set of subtrees to delete and get sorted elements(so search can stop earlier)
+    Owned<IPropertyTreeIterator> monthIter = root->getElements("*");
+    Owned<IException> innerException; //only first exception record/reported
+    ForEach(*monthIter)
+    {
+        IPropertyTree & monthPT = monthIter->query();
+        if (year==0)
+            deleteXpathList.push_back(monthPT.queryName());
+        else
+        {
+            unsigned msgYear = 0, msgMonth = 0;
+            const char *p = monthPT.queryName(); // should be in format 'myyyydd'
+            if (*p++ == 'm')
+            {
+                msgYear = readDigits(p, 4, false);
+                msgMonth = readDigits(p, 2, false);
+            }
+            if (msgYear == 0 || msgMonth == 0)
+            {
+                if (!innerException)
+                    innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT " is invalid: %s", monthPT.queryName()));
+                continue;
+            }
+            if (msgYear > year)
+                continue;
+            if (msgYear < year)
+                deleteXpathList.push_back(monthPT.queryName());
+            else
+            {
+                // msgYear matches year in this section
+                if (msgMonth > month)
+                    continue;
+                else if (msgMonth < month)
+                    deleteXpathList.push_back(monthPT.queryName());
+                else // msgMonth==month
+                {
+                    Owned<IPropertyTreeIterator> dayIter = monthPT.getElements("*");
+                    ForEach(*dayIter)
+                    {
+                        IPropertyTree & dayPT = dayIter->query();
+                        unsigned msgDay = 0;
+                        const char * d = dayPT.queryName();
+                        if (*d++ == 'd')
+                            msgDay = readDigits(d, 2);
+                        if (msgDay == 0)
+                        {
+                            if (!innerException)
+                                innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT "/%s is invalid: %s", monthPT.queryName(), dayPT.queryName()));
+                            continue;
+                        }
+                        if (day && (msgDay >= day))
+                            continue;
+
+                        VStringBuffer xpath("%s/%s", monthPT.queryName(), dayPT.queryName());
+                        deleteXpathList.push_back(xpath.str());
+                    }
+                }
+            }
+        }
+    }
+    unsigned count = 0;
+    for (auto & xpath: deleteXpathList)
+    {
+        if (root->removeProp(xpath.c_str()));
+            ++count;
+    }
+
+    if (innerException) // allow items to be deleted even if there is an exception
+        throw innerException.getClear();
+
+    return count;
+}
diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp
new file mode 100644
index 00000000000..2eb0af56edd
--- /dev/null
+++ b/dali/base/sysinfologger.hpp
@@ -0,0 +1,92 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#ifndef SYSINFOLOGGER
+#define SYSINFOLOGGER
+
+#include "jlog.hpp"
+#include "jutil.hpp"
+
+#ifdef DALI_EXPORT
+ #define SYSINFO_API DECL_EXPORT
+#else
+ #define SYSINFO_API DECL_IMPORT
+#endif
+
+interface ISysInfoLoggerMsg
+{
+    virtual bool queryIsHidden() const = 0;
+    virtual timestamp_type queryTimeStamp() const = 0;
+    virtual const char * querySource() const = 0;
+    virtual LogMsgCode queryLogMsgCode() const = 0;
+    virtual LogMsgAudience queryAudience() const = 0;
+    virtual LogMsgClass queryClass() const = 0;
+    virtual unsigned __int64 queryLogMsgId() const = 0;
+    virtual const char * queryMsg() const = 0;
+};
+
+interface IConstSysInfoLoggerMsgFilter : public IInterface
+{
+    virtual bool hasDateRange() const = 0;
+    virtual bool isInDateRange(timestamp_type ts) const = 0;
+    virtual bool queryHiddenOnly() const = 0;
+    virtual bool queryVisibleOnly() const = 0;
+    virtual timestamp_type queryMatchTimeStamp() const = 0;
+    virtual unsigned queryStartYear() const = 0;
+    virtual unsigned queryStartMonth() const = 0;
+    virtual unsigned queryStartDay() const = 0;
+    virtual unsigned queryEndYear() const = 0;
+    virtual unsigned queryEndMonth() const = 0;
+    virtual unsigned queryEndDay() const = 0;
+    virtual const char * queryMatchSource() const = 0;
+    virtual LogMsgCode queryMatchCode() const = 0;
+    virtual LogMsgAudience queryMatchAudience() const = 0;
+    virtual LogMsgClass queryMatchClass() const = 0;
+    virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const = 0;
+};
+
+interface ISysInfoLoggerMsgFilter : extends IConstSysInfoLoggerMsgFilter
+{
+    virtual void setHiddenOnly() = 0;
+    virtual void setVisibleOnly() = 0;
+    virtual void setMatchTimeStamp(timestamp_type ts) = 0;
+    virtual void setMatchSource(const char * source) = 0;
+    virtual void setMatchCode(LogMsgCode code) = 0;
+    virtual void setMatchAudience(LogMsgAudience audience) = 0;
+    virtual void setMatchMsgClass(LogMsgClass msgClass) = 0;
+    virtual void setMatchMsgId(unsigned __int64 msgId) = 0;
+    virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) = 0;
+    virtual void setOlderThanDate(unsigned year, unsigned month, unsigned day) = 0;
+};
+
+typedef IIteratorOf<ISysInfoLoggerMsg> ISysInfoLoggerMsgIterator;
+
+SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr);
+SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr);
+SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr);
+SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter);
+
+SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts);
+SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter);
+SYSINFO_API bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr);
+SYSINFO_API unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter, const char *source=nullptr);
+SYSINFO_API bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr);
+SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter);
+SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr);
+SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source=nullptr);
+
+#endif
diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp
index ea9f998bad3..428c18a29b9 100644
--- a/ecl/hthor/hthor.cpp
+++ b/ecl/hthor/hthor.cpp
@@ -7738,6 +7738,10 @@ void CHThorWSCBaseActivity::init()
         JBASE64_Encode(uidpair.str(), uidpair.length(), authToken, false);
     }
     soapTraceLevel = agent.queryWorkUnit()->getDebugValueInt("soapTraceLevel", 1);
+    StringBuffer soapSepStr;
+    StringBufferAdaptor soapSepAdaptor(soapSepStr);
+    agent.queryWorkUnit()->getDebugValue("soapLogSepString", soapSepAdaptor);
+    setSoapSepString(soapSepStr.str());
 }
 
 //---------------------------------------------------------------------------
diff --git a/esp/src/eclwatch/WUDetailsWidget.js b/esp/src/eclwatch/WUDetailsWidget.js
index c7d2c331af0..411bba32c52 100644
--- a/esp/src/eclwatch/WUDetailsWidget.js
+++ b/esp/src/eclwatch/WUDetailsWidget.js
@@ -117,6 +117,16 @@ define([
             this.emailFrom = registry.byId(this.id + "EmailFrom");
             this.emailSubject = registry.byId(this.id + "EmailSubject");
             this.emailBody = registry.byId(this.id + "EmailBody");
+
+            //Zap LogFilters
+            this.logFilterStartDateTime = dom.byId(this.id + "StartDateTime");
+            this.logFilterStartDate = registry.byId(this.id + "StartDate");
+            this.logFilterStartTime = registry.byId(this.id + "StartTime");
+            this.logFilterEndDateTime = dom.byId(this.id + "EndDateTime");
+            this.logFilterEndDate = registry.byId(this.id + "EndDate");
+            this.logFilterEndTime = registry.byId(this.id + "EndTime");
+            this.logFilterRelativeTimeRangeBuffer = registry.byId(this.id + "RelativeTimeRangeBuffer");
+
             this.protected = registry.byId(this.id + "Protected");
             this.infoGridWidget = registry.byId(this.id + "InfoContainer");
             this.zapDialog = registry.byId(this.id + "ZapDialog");
@@ -146,14 +156,33 @@ define([
             this.checkThorLogStatus();
         },
 
+        formatLogFilterDateTime: function (dateField, timeField, dateTimeField) {
+            if (dateField.value.toString() !== "Invalid Date") {
+                const d = new Date(dateField.value);
+                const date = `${d.getFullYear()}-${(d.getMonth() < 9 ? "0" : "") + parseInt(d.getMonth() + 1, 10)}-${d.getDate()}`;
+                const time = timeField.value.toString().replace(/.*1970\s(\S+).*/, "$1");
+                dateTimeField.value = `${date}T${time}.000Z`;
+            }
+        },
+
         _onSubmitDialog: function () {
             var context = this;
             var includeSlaveLogsCheckbox = this.includeSlaveLogsCheckbox.get("checked");
+            if (this.logFilterRelativeTimeRangeBuffer.value !== "") {
+                this.logFilterEndDate.required = "";
+                this.logFilterStartDate.required = "";
+            }
             if (this.zapForm.validate()) {
                 //WUCreateAndDownloadZAPInfo is not a webservice so relying on form to submit.
                 //Server treats "on" and '' as the same thing.
                 this.includeSlaveLogsCheckbox.set("value", includeSlaveLogsCheckbox ? "on" : "off");
+
+                // Log Filters
+                this.formatLogFilterDateTime(this.logFilterStartDate, this.logFilterStartTime, this.logFilterStartDateTime);
+                this.formatLogFilterDateTime(this.logFilterEndDate, this.logFilterEndTime, this.logFilterEndDateTime);
+
                 this.zapForm.set("action", "/WsWorkunits/WUCreateAndDownloadZAPInfo");
+
                 this.zapDialog.hide();
                 this.checkThorLogStatus();
                 if (this.logAccessorMessage !== "") {
diff --git a/esp/src/eclwatch/css/hpcc.css b/esp/src/eclwatch/css/hpcc.css
index 26559935861..e6d0e6f6f0b 100644
--- a/esp/src/eclwatch/css/hpcc.css
+++ b/esp/src/eclwatch/css/hpcc.css
@@ -74,6 +74,10 @@ form li label {
     padding-top: 4px;
 }
 
+.dijitDialogPaneContent {
+    overflow-x: hidden !important;
+}
+
 .dijitDialogPaneContent form li label {
     float: left;
     width: 25%;
diff --git a/esp/src/eclwatch/templates/WUDetailsWidget.html b/esp/src/eclwatch/templates/WUDetailsWidget.html
index a56d36ec15c..ef58752e1a6 100644
--- a/esp/src/eclwatch/templates/WUDetailsWidget.html
+++ b/esp/src/eclwatch/templates/WUDetailsWidget.html
@@ -183,7 +183,7 @@ <h2>
     <div id="${id}ZapDialog" data-dojo-type="dijit.Dialog" title="${i18n.ZippedAnalysisPackage}">
         <div class="dijitDialogPaneContentArea">
             <div id="${id}ZapForm" style="width:460px;" method="post" enctype="application/x-www-form-urlencoded" data-dojo-type="dijit.form.Form">
-                <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
+                <div data-dojo-props="cols:2" style="width:410px" data-dojo-type="hpcc.TableContainer">
                     <input id="${id}ZapName" title="${i18n.FileName}:" name="ZAPFileName" colspan="2" data-dojo-props="trim: true," data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}ZapWUID" title="${i18n.WUID}:" name="Wuid" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}BuildVersion" title="${i18n.ESPBuildVersion}:" name="BuildVersion" colspan="2" data-dojo-props="trim: true, readonly: true," data-dojo-type="dijit.form.TextBox" />
@@ -199,8 +199,41 @@ <h2>
                     <input id="${id}EmailFrom" title="${i18n.EmailFrom}:" name="EmailFrom" colspan="2" data-dojo-props="trim:true, placeHolder:'See Configuration Manager.'" data-dojo-type="dijit.form.TextBox" />
                     <input id="${id}EmailSubject" title="${i18n.EmailSubject}:" name="EmailSubject" colspan="2" data-dojo-props="trim:true" data-dojo-type="dijit.form.ValidationTextBox" required="false" />
                     <input id="${id}EmailBody" title="${i18n.EmailBody}:" name="EmailBody" cols="22" colspan="2" data-dojo-type="dijit.form.SimpleTextarea" />
+                    <input id="${id}StartDateTime" name="LogFilter_AbsoluteTimeRange_StartDate" type="hidden" />
+                    <input id="${id}StartDate" title="${i18n.FromDate}:" name="StartDate" data-dojo-props="trim: true" required="required" data-dojo-type="dijit.form.DateTextBox" />
+                    <input id="${id}StartTime" title="" name="StartTime" data-dojo-props="trim: true" value="T07:30:00" data-dojo-type="dijit.form.TimeTextBox" />
+                    <input id="${id}EndDateTime" name="LogFilter_AbsoluteTimeRange_EndDate" type="hidden" />
+                    <input id="${id}EndDate" title="${i18n.ToDate}:" name="EndDate" data-dojo-props="trim: true" required="required" data-dojo-type="dijit.form.DateTextBox" />
+                    <input id="${id}EndTime" title="" name="EndTime" data-dojo-props="trim: true" value="T19:30:00" data-dojo-type="dijit.form.TimeTextBox" />
+                    <input id="${id}RelativeTimeRangeBuffer" title="${i18n.RelativeTimeRange}" name="LogFilter_RelativeTimeRangeBuffer" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <input id="${id}LineLimit" title="${i18n.LogLineLimit}" name="LogFilter_LineLimit" colspan="2" value="10000" data-dojo-type="dijit.form.TextBox" />
+                    <input id="${id}LineStartFrom" title="${i18n.LogLineStartFrom}" name="LogFilter_LineStartFrom" value="0" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <select id="${id}SelectColumnMode" title="${i18n.ColumnMode}" name="LogFilter_SelectColumnMode" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option value="0">MIN</option>
+                        <option selected="selected" value="1">DEFAULT</option>
+                        <option value="2">ALL</option>
+                    </select>
+                    <select id="${id}LogFilterFormat" title="${i18n.LogFormat}" name="LogFilter_Format" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option selected="selected" value="csv">CSV</option>
+                        <option value="json">JSON</option>
+                        <option value="xml">XML</option>
+                    </select>
+                    <input id="${id}WildcardFilter" title="${i18n.WildcardFilter}" name="LogFilter_WildcardFilter" colspan="2" data-dojo-type="dijit.form.TextBox" />
+                    <select id="${id}sortByTimeDirection" title="${i18n.Sort} ${i18n.TimeStamp}" name="LogFilter_sortByTimeDirection" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option value="0">ASC</option>
+                        <option selected="selected" value="1">DESC</option>
+                    </select>
+                    <select id="${id}LogEventType" title="${i18n.LogEventType}" name="LogFilter_LogEventType" colspan="2" data-dojo-type="dijit.form.Select">
+                        <option selected="selected" value="ALL">All</option>
+                        <option value="DIS">Disaster</option>
+                        <option value="ERR">Error</option>
+                        <option value="WRN">Warning</option>
+                        <option value="INF">Information</option>
+                        <option value="PRO">Progress</option>
+                        <option value="MET">Metric</option>
+                    </select>
                 </div>
-                <div class="dijitDialogPaneActionBar">
+                <div style="width:410px" class="dijitDialogPaneActionBar">
                     <button id="${id}onZapSubmit" data-dojo-attach-event="onClick:_onSubmitDialog" type="submit" data-dojo-type="dijit.form.Button">${i18n.Apply}</button>
                     <button style="float:left" data-dojo-attach-event="onClick:_onCancelDialog" data-dojo-type="dijit.form.Button">${i18n.Cancel}</button>
                 </div>
diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json
index 262f9708232..aa0994e6469 100644
--- a/esp/src/package-lock.json
+++ b/esp/src/package-lock.json
@@ -18,7 +18,7 @@
         "@hpcc-js/chart": "2.84.1",
         "@hpcc-js/codemirror": "2.63.0",
         "@hpcc-js/common": "2.72.0",
-        "@hpcc-js/comms": "2.96.1",
+        "@hpcc-js/comms": "2.97.0",
         "@hpcc-js/dataflow": "8.1.7",
         "@hpcc-js/eclwatch": "2.75.3",
         "@hpcc-js/graph": "2.86.0",
@@ -2144,9 +2144,9 @@
       }
     },
     "node_modules/@hpcc-js/comms": {
-      "version": "2.96.1",
-      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.96.1.tgz",
-      "integrity": "sha512-38vIe8foZa5fYtrj65oeWyYWUDZmQTbKetHG5HXWZWMu0Lfmln8uG5/J7mO0ilw3ls2oZj7xOk5T/4xvg7v43w==",
+      "version": "2.97.0",
+      "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.97.0.tgz",
+      "integrity": "sha512-1AqZoYNDhb/zRLc/ZSj8eIrpOab1fUzMXFZYVYJvbDgO6dcwDn5cFMG+Y3g5m9Z/sc48E50S9lTZFFqQFZQOGg==",
       "dependencies": {
         "@hpcc-js/ddl-shim": "^2.21.0",
         "@hpcc-js/util": "^2.52.0",
diff --git a/esp/src/package.json b/esp/src/package.json
index f3f5270f0e7..2676b215461 100644
--- a/esp/src/package.json
+++ b/esp/src/package.json
@@ -44,7 +44,7 @@
     "@hpcc-js/chart": "2.84.1",
     "@hpcc-js/codemirror": "2.63.0",
     "@hpcc-js/common": "2.72.0",
-    "@hpcc-js/comms": "2.96.1",
+    "@hpcc-js/comms": "2.97.0",
     "@hpcc-js/dataflow": "8.1.7",
     "@hpcc-js/eclwatch": "2.75.3",
     "@hpcc-js/graph": "2.86.0",
diff --git a/esp/src/src-react/components/Logs.tsx b/esp/src/src-react/components/Logs.tsx
index 0ec5a9b4699..a0ef17ee26b 100644
--- a/esp/src/src-react/components/Logs.tsx
+++ b/esp/src/src-react/components/Logs.tsx
@@ -3,7 +3,8 @@ import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluen
 import { GetLogsExRequest, LogaccessService, TargetAudience, LogType } from "@hpcc-js/comms";
 import { Level, scopedLogger } from "@hpcc-js/util";
 import nlsHPCC from "src/nlsHPCC";
-import { logColor, wuidToDate, wuidToTime } from "src/Utility";
+import { logColor, timestampToDate, wuidToDate, wuidToTime } from "src/Utility";
+import { useLoggingEngine } from "../hooks/platform";
 import { HolyGrail } from "../layouts/HolyGrail";
 import { pushParams } from "../util/history";
 import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid";
@@ -110,28 +111,49 @@ export const Logs: React.FunctionComponent<LogsProps> = ({
 
     const now = React.useMemo(() => new Date(), []);
 
+    const loggingEngine = useLoggingEngine();
+
     //  Grid ---
     const columns = React.useMemo((): FluentColumns => {
-        return {
-            timestamp: { label: nlsHPCC.TimeStamp, width: 140, sortable: false, },
-            message: { label: nlsHPCC.Message, width: 600, sortable: false, },
-            components: { label: nlsHPCC.ContainerName, width: 150, sortable: false },
-            instance: { label: nlsHPCC.PodName, width: 150, sortable: false },
-            audience: { label: nlsHPCC.Audience, width: 60, sortable: false, },
-            class: {
-                label: nlsHPCC.Class, width: 40, sortable: false,
-                formatter: level => {
-                    const colors = logColor(levelMap(level));
-                    const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground };
-                    return <span style={styles}>{level}</span>;
-                }
+        let retVal = {
+            timestamp: {
+                label: nlsHPCC.TimeStamp, width: 140, sortable: false,
+                formatter: ts => {
+                    if (ts) {
+                        if (ts.indexOf(":") < 0) {
+                            return timestampToDate(ts).toISOString();
+                        }
+                        return new Date(ts).toISOString();
+                    }
+                },
             },
-            workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, },
-            processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, },
-            logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
-            threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
+            message: { label: nlsHPCC.Message, width: 600, sortable: false, },
         };
-    }, [wuid]);
+        if (loggingEngine === "grafanacurl") {
+            retVal = Object.assign(retVal, {
+                pod: { label: nlsHPCC.PodName, width: 150, sortable: false },
+            });
+        } else {
+            retVal = Object.assign(retVal, {
+                instance: { label: nlsHPCC.PodName, width: 150, sortable: false },
+                components: { label: nlsHPCC.ContainerName, width: 150, sortable: false },
+                audience: { label: nlsHPCC.Audience, width: 60, sortable: false, },
+                class: {
+                    label: nlsHPCC.Class, width: 40, sortable: false,
+                    formatter: level => {
+                        const colors = logColor(levelMap(level));
+                        const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground };
+                        return <span style={styles}>{level}</span>;
+                    }
+                },
+                workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, },
+                processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, },
+                logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, },
+                threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, },
+            });
+        }
+        return retVal;
+    }, [loggingEngine, wuid]);
 
     const copyButtons = useCopyButtons(columns, selection, "logaccess");
 
diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx
index ff45b2a7f6d..a64847aff70 100644
--- a/esp/src/src-react/components/Queries.tsx
+++ b/esp/src/src-react/components/Queries.tsx
@@ -120,8 +120,8 @@ export const Queries: React.FunctionComponent<QueriesProps> = ({
                 headerTooltip: nlsHPCC.ErrorWarnings,
                 width: 16,
                 sortable: false,
-                formatter: (error) => {
-                    if (error > 0) {
+                formatter: (error, row) => {
+                    if (row.ErrorCount > 0) {
                         return <Icon iconName="Warning" />;
                     }
                     return "";
@@ -133,8 +133,9 @@ export const Queries: React.FunctionComponent<QueriesProps> = ({
                 headerTooltip: nlsHPCC.MixedNodeStates,
                 width: 16,
                 sortable: false,
-                formatter: (mixed) => {
-                    if (mixed === true) {
+                formatter: (mixed, row) => {
+                    const mixedStates = row.Clusters.ClusterQueryState[0]?.MixedNodeStates ?? false;
+                    if (mixedStates === true) {
                         return <Icon iconName="Error" />;
                     }
                     return "";
@@ -144,8 +145,8 @@ export const Queries: React.FunctionComponent<QueriesProps> = ({
                 headerIcon: "SkypeCircleCheck",
                 headerTooltip: nlsHPCC.Active,
                 width: 16,
-                formatter: (activated) => {
-                    if (activated === true) {
+                formatter: (activated, row) => {
+                    if (row.Activated === true) {
                         return <Icon iconName="SkypeCircleCheck" />;
                     }
                     return "";
diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts
index 66dac5d9487..4bc86caa2c4 100644
--- a/esp/src/src-react/hooks/platform.ts
+++ b/esp/src/src-react/hooks/platform.ts
@@ -2,7 +2,7 @@ import * as React from "react";
 import { Octokit } from "octokit";
 import { useConst } from "@fluentui/react-hooks";
 import { scopedLogger } from "@hpcc-js/util";
-import { Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms";
+import { LogaccessService, Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms";
 import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session";
 import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo";
 import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore";
@@ -10,6 +10,8 @@ import { Palette } from "@hpcc-js/common";
 
 const logger = scopedLogger("src-react/hooks/platform.ts");
 
+export const service = new LogaccessService({ baseUrl: "" });
+
 declare const dojoConfig;
 
 export function useBuildInfo(): [BuildInfo, { isContainer: boolean, currencyCode: string, opsCategory: string }] {
@@ -205,4 +207,14 @@ export function useModernMode(): {
     }, [modernMode, sessionStore, userStore]);
 
     return { modernMode, setModernMode };
-} 
\ No newline at end of file
+}
+
+export function useLoggingEngine(): string {
+    const [loggingEngine, setLoggingEngine] = React.useState("");
+
+    React.useEffect(() => {
+        service.GetLogAccessInfo({}).then(response => setLoggingEngine(response.RemoteLogManagerType ?? ""));
+    }, []);
+
+    return loggingEngine;
+}
\ No newline at end of file
diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts
index cc78350ed6d..4b70d873129 100644
--- a/esp/src/src/Utility.ts
+++ b/esp/src/src/Utility.ts
@@ -1242,6 +1242,21 @@ export function format(labelTpl, obj) {
         .join("\n")
         ;
 }
+
+const TEN_TRILLION = 10000000000000;
+export function nanosToMillis(timestamp: number): number {
+    if (timestamp > TEN_TRILLION) {
+        return Math.round(timestamp / 1000000);
+    } else {
+        return timestamp;
+    }
+}
+
+export function timestampToDate(timestamp: number): Date {
+    const millis = nanosToMillis(timestamp);
+    return new Date(millis);
+}
+
 const theme = getTheme();
 const { semanticColors } = theme;
 
diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp
index 9d2680aa8ee..192e58a5670 100644
--- a/roxie/ccd/ccdmain.cpp
+++ b/roxie/ccd/ccdmain.cpp
@@ -888,6 +888,12 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1);
         roxiemem::setMemTraceLevel(topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1));
         soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1);
+        if (topology->hasProp("@soapLogSepString"))
+        {
+            StringBuffer tmpSepString;
+            topology->getProp("@soapLogSepString", tmpSepString);
+            setSoapSepString(tmpSepString.str());
+        }
         miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0);
         traceRemoteFiles = topology->getPropBool("@traceRemoteFiles", false);
         testAgentFailure = topology->getPropInt("expert/@testAgentFailure", testAgentFailure);
diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp
index c2ba7d12313..76a4069b61d 100644
--- a/system/jlib/jdebug.cpp
+++ b/system/jlib/jdebug.cpp
@@ -835,6 +835,54 @@ static struct CNtKernelInformation
 } NtKernelFunctions;
 #endif
 
+//===========================================================================
+
+#ifndef _WIN32
+
+static std::atomic<bool> gatheredGroup{false};
+static StringAttr cgroup;
+static CriticalSection csgroupCs;
+static const char * queryCGroup()
+{
+    if (!gatheredGroup)
+    {
+        CriticalBlock block(csgroupCs);
+        if (!gatheredGroup)
+        {
+            StringBuffer contents;
+            if (loadBinaryFile(contents, "/proc/self/cgroup", false))
+            {
+                auto processLine = [](size_t len, const char * ln)
+                {
+                    //Note ln points at the start of the line, but the line is not null terminated
+                    switch (*ln)
+                    {
+                    case '0':
+                        if (strncmp(ln, "0::/", 4) == 0)
+                        {
+                            //Format is 0::/<cgroup>
+                            //If not running in a container the "cgroup" may be something like user.slice/user-1000.slice/user@1000.service/....
+                            //If so ignore because it is not a real cgroup
+                            if (!memchr(ln+4, '/', len-4))
+                                cgroup.set(ln+4, len-4);
+                        }
+                        break;
+                    }
+                    //Some systems with version 1 cgroups have <n>:cpu,cpuacct:/<cgroup>
+                    const char  * match = (const char *)jmemmem(len, ln, 14, ":cpu,cpuacct:/");
+                    if (match)
+                        cgroup.set(match+14, (ln + len) - (match + 14));
+                };
+
+                processLines(contents, processLine);
+            }
+            gatheredGroup = true;
+        }
+    }
+    return cgroup.get();
+}
+
+#endif
 
 //===========================================================================
 
@@ -912,6 +960,9 @@ SystemProcessInfo SystemProcessInfo::operator - (const SystemProcessInfo & rhs)
     result.activeDataMemory = activeDataMemory - rhs.activeDataMemory;
     result.majorFaults = majorFaults - rhs.majorFaults;
     result.numThreads = numThreads - rhs.numThreads;
+    result.numPeriods = numPeriods - rhs.numPeriods;
+    result.numThrottledPeriods = numThrottledPeriods - rhs.numThrottledPeriods;
+    result.timeThrottledNs = timeThrottledNs - rhs.timeThrottledNs;
     return result;
 }
 
@@ -1014,7 +1065,7 @@ bool ProcessInfo::update(unsigned flags)
         if (loadBinaryFile(contents, "/proc/self/status", false))
         {
             contextSwitches = 0;
-            auto processLine = [this](const char * cur)
+            auto processLine = [this](size_t len, const char * cur)
             {
                 __uint64 value;
                 switch (*cur)
@@ -1089,7 +1140,7 @@ bool SystemInfo::update(unsigned flags)
     StringBuffer contents;
     if (loadBinaryFile(contents, "/proc/stat", false))
     {
-        auto processLine = [this](const char * ln)
+        auto processLine = [this](size_t len, const char * ln)
         {
             switch (*ln)
             {
@@ -1124,6 +1175,52 @@ bool SystemInfo::update(unsigned flags)
         processLines(contents, processLine);
     }
 
+    auto processLine = [this](size_t len, const char * ln)
+    {
+        switch (*ln)
+        {
+        case 'n':
+            if (strncmp(ln, "nr_periods ", 11) == 0)
+                numPeriods = strtod(ln+11, nullptr);
+            else if (strncmp(ln, "nr_throttled ", 13) == 0)
+                numThrottledPeriods = strtod(ln+13, nullptr);
+            break;
+        case 't':
+            if (strncmp(ln, "throttled_usec ", 15) == 0)
+                timeThrottledNs = strtod(ln+15, nullptr) * 1000;
+            else if (strncmp(ln, "throttled_time ", 15) == 0)
+                timeThrottledNs = strtod(ln+15, nullptr);
+            break;
+        }
+    };
+
+    bool done = false;
+    const char * cgroup = queryCGroup();
+    if (cgroup)
+    {
+        //Version 2 of cgroups has the information in cgroup/<cgroup>
+        VStringBuffer filename("/sys/fs/cgroup/%s/cpu.stat", cgroup);
+        if (loadBinaryFile(contents.clear(), filename.str(), false))
+        {
+            processLines(contents, processLine);
+            done = true;
+        }
+        else
+        {
+            //Some systems with version 1 cgroups have the information in /sys/fs/cgroup/cpu/<cgroup>/cpu.stat
+            filename.clear().appendf("/sys/fs/cgroup/cpu/%s/cpu.stat", cgroup);
+            if (loadBinaryFile(contents.clear(), filename.str(), false))
+            {
+                processLines(contents, processLine);
+                done = true;
+            }
+        }
+    }
+
+    //If the version 2 file was not found look for ther version 1 information in cgroup/cpu
+    if (!done && loadBinaryFile(contents.clear(), "/sys/fs/cgroup/cpu/cpu.stat", false))
+        processLines(contents, processLine);
+
     return true;
 #endif
 }
@@ -2521,9 +2618,16 @@ class CExtendedStats  // Disk network and cpu stats
         }
         if (totalcpu)
         {
-            if (out.length()&&(out.charAt(out.length()-1)!=' '))
-                out.append(' ');
+            ensureSeparator(out, ' ');
             out.appendf("CPU: usr=%d sys=%d iow=%d idle=%d", deltacpu.getUserPercent(), deltacpu.getSystemPercent(), deltacpu.getIoWaitPercent(), deltacpu.getIdlePercent());
+
+            __uint64 periods = deltacpu.getNumPeriods();
+            if (periods)
+            {
+                unsigned throttling = deltacpu.getNumThrottledPeriods() * 100 / periods;
+                __uint64 timeThrottledNs = deltacpu.getTimeThrottledNs();
+                out.appendf(" thr=%u%% thrns=%llu", throttling, timeThrottledNs);
+            }
         }
         return true;
     }
diff --git a/system/jlib/jdebug.hpp b/system/jlib/jdebug.hpp
index 0a8a0dacfcc..a4d5d41c705 100644
--- a/system/jlib/jdebug.hpp
+++ b/system/jlib/jdebug.hpp
@@ -446,6 +446,10 @@ class jlib_decl SystemProcessInfo
     unsigned getSystemPercent() const;
     unsigned getUserPercent() const;
 
+    __uint64 getNumPeriods() const { return numPeriods; }
+    __uint64 getNumThrottledPeriods() const { return numThrottledPeriods; }
+    __uint64 getTimeThrottledNs() const { return timeThrottledNs; }
+
     __uint64 getTotal() const { return user + system + idle + iowait; }
 protected:
     __uint64 user = 0;      // user time in jiffies (~`1/100s)
@@ -461,6 +465,9 @@ class jlib_decl SystemProcessInfo
     __uint64 activeDataMemory = 0;
     __uint64 majorFaults = 0;
     __uint64 numThreads = 0;
+    __uint64 numPeriods = 0;
+    __uint64 numThrottledPeriods = 0;
+    __uint64 timeThrottledNs = 0;
 };
 
 class jlib_decl ProcessInfo : public SystemProcessInfo
diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp
index 24e6029df94..f9749a067f4 100644
--- a/system/jlib/jfcmp.hpp
+++ b/system/jlib/jfcmp.hpp
@@ -230,6 +230,7 @@ class jlib_decl CFcmpExpander : public CExpanderBase
     size32_t outlen;
     size32_t bufalloc;
     const size32_t *in = nullptr;
+    const void * original = nullptr;
 
 public:
     CFcmpExpander()
@@ -247,6 +248,7 @@ class jlib_decl CFcmpExpander : public CExpanderBase
 
     virtual size32_t init(const void *blk)
     {
+        original = blk;
         const size32_t *expsz = (const size32_t *)blk;
         outlen = *expsz;
         in = (expsz+1);
diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp
index 6a854c39fbf..930b38fa02d 100644
--- a/system/jlib/jlz4.cpp
+++ b/system/jlib/jlz4.cpp
@@ -264,6 +264,13 @@ class jlib_decl CLZ4Expander : public CFcmpExpander
         size32_t written;
         if (szchunk+totalExpanded<outlen)
         {
+            if (unlikely(szchunk == 0))
+            {
+                //Special case this corruption - otherwise it enters an infinite loop
+                VStringBuffer msg("Unexpected zero length block at block offset %u", (size32_t)((const byte *)in - (const byte *)original));
+                throwUnexpectedX(msg.str());
+            }
+
             //All but the last block are compressed (see expand() function above).
             //Slightly concerning there always has to be one trailing byte for this to work!
             size32_t maxOut = target.capacity();
@@ -286,7 +293,10 @@ class jlib_decl CLZ4Expander : public CFcmpExpander
 
                 //Sanity check to catch corrupt lz4 data that always returns an error.
                 if (maxOut > outlen)
-                    throwUnexpected();
+                {
+                    VStringBuffer msg("Decompression expected max %u bytes, but now %u at block offset %u", outlen, maxOut, (size32_t)((const byte *)in - (const byte *)original));
+                    throwUnexpectedX(msg.str());
+                }
 
                 maxOut += szchunk; // Likely to quickly approach the actual expanded size
                 target.clear();
diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp
index 6ff771eade6..6401657a7a9 100644
--- a/system/jlib/jlzw.cpp
+++ b/system/jlib/jlzw.cpp
@@ -2118,7 +2118,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
         {
             unsigned code = e->errorCode();
             StringBuffer msg;
-            e->errorMessage(msg).appendf(" at position %llu of %llu", nextExpansionPos, trailer.indexPos);
+            e->errorMessage(msg).appendf(" at uncompressed position %llu block %u of %llu", nextExpansionPos, curblocknum, trailer.indexPos);
             e->Release();
             throw makeStringException(code, msg.str());
         }
@@ -2157,37 +2157,48 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface
 
     void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize, offset_t compressedPos)
     {
-        size32_t rs = trailer.recordSize;
-        if (rs) { // diff expand
-            const byte *src = (const byte *)compbuf;
-            byte *dst = (byte *)expbuf.reserve(expsize);
-            if (expsize) {
-                assertex(expsize>=rs);
-                memcpy(dst,src,rs);
-                dst += rs;
-                src += rs;
-                expsize -= rs;
-                while (expsize) {
+        try
+        {
+            size32_t rs = trailer.recordSize;
+            if (rs) { // diff expand
+                const byte *src = (const byte *)compbuf;
+                byte *dst = (byte *)expbuf.reserve(expsize);
+                if (expsize) {
                     assertex(expsize>=rs);
-                    src += DiffExpand(src, dst, dst-rs, rs);
-                    expsize -= rs;
+                    memcpy(dst,src,rs);
                     dst += rs;
+                    src += rs;
+                    expsize -= rs;
+                    while (expsize) {
+                        assertex(expsize>=rs);
+                        src += DiffExpand(src, dst, dst-rs, rs);
+                        expsize -= rs;
+                        dst += rs;
+                    }
                 }
             }
-        }
-        else { // lzw or fastlz or lz4
-            assertex(expander.get());
-            size32_t exp = expander->expandFirst(expbuf, compbuf);
-            if (exp == 0)
-            {
-                unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf);
-                if (numZeros >= 16)
-                    throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros);
-            }
+            else { // lzw or fastlz or lz4
+                assertex(expander.get());
+                size32_t exp = expander->expandFirst(expbuf, compbuf);
+                if (exp == 0)
+                {
+                    unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf);
+                    if (numZeros >= 16)
+                        throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros);
+                }
 
-            startBlockPos = curblockpos;
-            nextExpansionPos = startBlockPos + exp;
-            fullBlockSize = expsize;
+                startBlockPos = curblockpos;
+                nextExpansionPos = startBlockPos + exp;
+                fullBlockSize = expsize;
+            }
+        }
+        catch (IException * e)
+        {
+            unsigned code = e->errorCode();
+            StringBuffer msg;
+            e->errorMessage(msg).appendf(" at compressed position %llu of %llu", compressedPos, trailer.indexPos);
+            e->Release();
+            throw makeStringException(code, msg.str());
         }
     }
 
diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp
index 1c3402d3318..da160d112e4 100644
--- a/system/jlib/jlzw.hpp
+++ b/system/jlib/jlzw.hpp
@@ -145,8 +145,8 @@ extern jlib_decl bool isCompressedFile(const char *filename);
 extern jlib_decl bool isCompressedFile(IFile *file);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL);
-extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4, IFEflags extraFlags=IFEnone);
 extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio, bool append, size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4);
+extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4, IFEflags extraFlags=IFEnone);
 
 #define COMPRESSEDFILECRC (~0U)
 
diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp
index 50951938ae5..7e581f2503a 100644
--- a/system/jlib/jstring.cpp
+++ b/system/jlib/jstring.cpp
@@ -2902,6 +2902,12 @@ void getSnakeCase(StringBuffer & out, const char * camelValue)
     }
 }
 
+void ensureSeparator(StringBuffer & out, char separator)
+{
+    if (out.length() && (out.charAt(out.length()-1) != separator))
+        out.append(separator);
+}
+
 /**
  * stristr - Case insensitive strstr()
  * @haystack: Where we will search for our @needle
@@ -2949,3 +2955,31 @@ const char * stristr (const char *haystack, const char *needle)
     }
     return nullptr;
 }
+
+
+const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle)
+{
+    if (lenNeedle == 0)
+        return haystack;
+
+    if (lenHaystack < lenNeedle)
+        return nullptr;
+
+    const char * search = (const char *)needle;
+    char first = *search;
+    if (lenNeedle == 1)
+        return memchr(haystack, first, lenHaystack);
+
+    const char * buffer = (const char *)haystack;
+    for (size_t i = 0; i <= lenHaystack - lenNeedle; i++)
+    {
+        //Special case the first character to avoid a function call each iteration.
+        if (buffer[i] == first)
+        {
+            if (memcmp(buffer + i + 1, search + 1, lenNeedle-1) == 0)
+                return buffer + i;
+        }
+    }
+
+    return nullptr;
+}
diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp
index b3fe7651daf..d61b7fb7a1f 100644
--- a/system/jlib/jstring.hpp
+++ b/system/jlib/jstring.hpp
@@ -632,11 +632,18 @@ void processLines(const StringBuffer & content, LineProcessor process)
     const char * cur = content;
     while (*cur)
     {
-        process(cur);
         const char * next = strchr(cur, '\n');
-        if (!next)
+        if (next)
+        {
+            if (next != cur)
+                process(next-cur, cur);
+            cur = next+1;
+        }
+        else
+        {
+            process(strlen(cur), cur);
             break;
-        cur = next+1;
+        }
     }
 }
 
@@ -646,5 +653,10 @@ extern jlib_decl void processOptionString(const char * options, optionCallback c
 
 extern jlib_decl const char * stristr(const char *haystack, const char *needle);
 extern jlib_decl void getSnakeCase(StringBuffer & out, const char * camelValue);
+//If the string has any characters, ensure the last character matches the separator
+extern jlib_decl void ensureSeparator(StringBuffer & out, char separator);
+
+//Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own
+extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle);
 
 #endif
diff --git a/system/jlib/jtime.cpp b/system/jlib/jtime.cpp
index 87ed2384b7b..58dca7f0639 100644
--- a/system/jlib/jtime.cpp
+++ b/system/jlib/jtime.cpp
@@ -117,19 +117,6 @@ time_t timelocal(struct tm * local)
 
 #endif //__GNUC__
 
-static unsigned readDigits(char const * & str, unsigned numDigits)
-{
-    unsigned ret = 0;
-    while(numDigits--)
-    {
-        char c = *str++;
-        if(!isdigit(c))
-            throwError1(JLIBERR_BadlyFormedDateTime, str);
-        ret  = ret * 10 + (c - '0');
-    }
-    return ret;
-}
-
 static void checkChar(char const * & str, char required)
 {
     char c = *str++;
@@ -222,22 +209,32 @@ void CDateTime::set(time_t simple)
 
 void CDateTime::setString(char const * str, char const * * end, bool local)
 {
+    char const * beginstr = str; // save for error message
     if (!str||!*str) {
         clear();
         return;
     }
-    unsigned year = readDigits(str, 4);
-    checkChar(str, '-');
-    unsigned month = readDigits(str, 2);
-    checkChar(str, '-');
-    unsigned day = readDigits(str, 2);
-    checkChar(str, 'T');
-    unsigned hour = readDigits(str, 2);
-    checkChar(str, ':');
-    unsigned minute = readDigits(str, 2);
-    checkChar(str, ':');
-    unsigned sec = readDigits(str, 2);
-    unsigned nano = 0;
+    unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0, nano = 0;
+    try
+    {
+        year = readDigits(str, 4);
+        checkChar(str, '-');
+        month = readDigits(str, 2);
+        checkChar(str, '-');
+        day = readDigits(str, 2);
+        checkChar(str, 'T');
+        hour = readDigits(str, 2);
+        checkChar(str, ':');
+        minute = readDigits(str, 2);
+        checkChar(str, ':');
+        sec = readDigits(str, 2);
+    }
+    catch (IException * e)
+    {
+        e->Release();
+        throwError1(JLIBERR_BadlyFormedDateTime, beginstr);
+    }
+
     if(*str == '.')
     {
         unsigned digits;
@@ -256,26 +253,44 @@ void CDateTime::setString(char const * str, char const * * end, bool local)
 
 void CDateTime::setDateString(char const * str, char const * * end)
 {
-    unsigned year = readDigits(str, 4);
-    checkChar(str, '-');
-    unsigned month = readDigits(str, 2);
-    checkChar(str, '-');
-    unsigned day = readDigits(str, 2);
+    char const * beginstr = str; // save for error message
+    unsigned year = 0, month = 0, day = 0;
+    try
+    {
+        year = readDigits(str, 4);
+        checkChar(str, '-');
+        month = readDigits(str, 2);
+        checkChar(str, '-');
+        day = readDigits(str, 2);
+    }
+    catch (IException * e)
+    {
+        e->Release();
+        throwError1(JLIBERR_BadlyFormedDateTime, beginstr);
+    }
     if(end) *end = str;
     set(year, month, day, 0, 0, 0, 0, false);
 }
 
 void CDateTime::setTimeString(char const * str, char const * * end, bool local)
 {
-    unsigned year;
-    unsigned month;
-    unsigned day;
+    char const * beginstr = str; // save for error message
+    unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0;
     getDate(year, month, day, false);
-    unsigned hour = readDigits(str, 2);
-    checkChar(str, ':');
-    unsigned minute = readDigits(str, 2);
-    checkChar(str, ':');
-    unsigned sec = readDigits(str, 2);
+
+    try
+    {
+        hour = readDigits(str, 2);
+        checkChar(str, ':');
+        minute = readDigits(str, 2);
+        checkChar(str, ':');
+        sec = readDigits(str, 2);
+    }
+    catch (IException * e)
+    {
+        e->Release();
+        throwError1(JLIBERR_BadlyFormedDateTime, beginstr);
+    }
     unsigned nano = 0;
     if(*str == '.')
     {
@@ -668,10 +683,18 @@ void CScmDateTime::setString(const char * pstr)
     else if ((sign == '-') || (sign == '+'))
     {
         end++;
-        int delta = readDigits(end, 2);
-        if (*end++ != ':')
+        int delta = 0;
+        try
+        {
+            delta = readDigits(end, 2);
+            checkChar(end, ':');
+            delta = delta * 60 + readDigits(end, 2);
+        }
+        catch (IException * e)
+        {
+            e->Release();
             throwError1(JLIBERR_BadlyFormedDateTime, pstr);
-        delta = delta * 60 + readDigits(end, 2);
+        }
         if (sign == '-')
             delta = -delta;
         utcToLocalDelta = delta;
diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp
index e2013d8399e..d22fda4888d 100644
--- a/system/jlib/jutil.cpp
+++ b/system/jlib/jutil.cpp
@@ -3632,3 +3632,20 @@ void hold(const char *msg)
 }
 
 
+
+unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure)
+{
+    unsigned ret = 0;
+    while (numDigits--)
+    {
+        char c = *str++;
+        if (!isdigit(c))
+        {
+            if (throwOnFailure)
+                throw makeStringExceptionV(-1, "Invalid format (readDigits): %s", str);
+            return 0;
+        }
+        ret  = ret * 10 + (c - '0');
+    }
+    return ret;
+}
diff --git a/system/jlib/jutil.hpp b/system/jlib/jutil.hpp
index 07c81dbc2cd..4841d61761b 100644
--- a/system/jlib/jutil.hpp
+++ b/system/jlib/jutil.hpp
@@ -655,6 +655,8 @@ struct HPCCBuildInfo
 extern jlib_decl HPCCBuildInfo hpccBuildInfo;
 extern jlib_decl bool checkCreateDaemon(unsigned argc, const char * * argv);
 
+extern jlib_decl unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure = true);
+
 //Createpassword of specified length, containing UpperCaseAlphas, LowercaseAlphas, numerics and symbols
 extern jlib_decl const char * generatePassword(StringBuffer &pwd, int pwdLen);
 
diff --git a/testing/unittests/CMakeLists.txt b/testing/unittests/CMakeLists.txt
index 73571bee22d..59498fb0d2f 100644
--- a/testing/unittests/CMakeLists.txt
+++ b/testing/unittests/CMakeLists.txt
@@ -46,6 +46,7 @@ set (    SRCS
          accessmaptests.cpp
          fxpptests.cpp
          espapicmdtests.cpp
+         filetests.cpp
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp/fxpp/FragmentedXmlPullParser.cpp
          ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp/fxpp/FragmentedXmlAssistant.cpp
          ${CMAKE_BINARY_DIR}/generated/ws_loggingservice_esp.cpp
diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp
index 1472cad5b37..c10a7529d2d 100644
--- a/testing/unittests/dalitests.cpp
+++ b/testing/unittests/dalitests.cpp
@@ -37,6 +37,7 @@
 #include <math.h>
 
 #include "unittests.hpp"
+#include "sysinfologger.hpp"
 
 //#define COMPAT
 
@@ -3034,4 +3035,231 @@ class CFileNameNormalizeUnitTest : public CppUnit::TestFixture, CDfsLogicalFileN
 CPPUNIT_TEST_SUITE_REGISTRATION( CFileNameNormalizeUnitTest );
 CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CFileNameNormalizeUnitTest, "CFileNameNormalizeUnitTest" );
 
+#define SOURCE_COMPONENT_UNITTEST "sysinfologger-unittest"
+
+class DaliSysInfoLoggerTester : public CppUnit::TestFixture
+{
+    /* Note: global messages will be written for dates between 2000-02-04 and 2000-02-05 */
+    /* Note: All global messages with time stamp before 2000-03-31 will be deleted */
+    CPPUNIT_TEST_SUITE(DaliSysInfoLoggerTester);
+        CPPUNIT_TEST(testInit);
+        CPPUNIT_TEST(testSysInfoLogger);
+    CPPUNIT_TEST_SUITE_END();
+
+    struct TestCase
+    {
+        LogMsgCategory cat;
+        LogMsgCode code;
+        bool hidden;
+        const char * dateTimeStamp;
+        const char * msg;
+    };
+
+    std::vector<TestCase> testCases =
+    {
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42301,
+            false,
+            "2000-02-03T10:01:22.342343",
+            "CSysInfoLogger Unit test message 1"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42302,
+            false,
+            "2000-02-03T12:03:42.114233",
+            "CSysInfoLogger Unit test message 2"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42303,
+            true,
+            "2000-02-03T14:02:13.678443",
+            "CSysInfoLogger Unit test message 3"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42304,
+            true,
+            "2000-02-03T16:05:18.8324832",
+            "CSysInfoLogger Unit test message 4"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42301,
+            false,
+            "2000-02-04T03:01:42.5754",
+            "CSysInfoLogger Unit test message 5"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42302,
+            false,
+            "2000-02-04T09:06:25.133132",
+            "CSysInfoLogger Unit test message 6"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42303,
+            false,
+            "2000-02-04T11:09:32.78439",
+            "CSysInfoLogger Unit test message 7"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42304,
+            true,
+            "2000-02-04T13:02:12.82821",
+            "CSysInfoLogger Unit test message 8"
+        },
+        {
+            LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail),
+            42304,
+            true,
+            "2000-02-04T18:32:11.23421",
+            "CSysInfoLogger Unit test message 9"
+        }
+    };
+
+    struct WrittenLogMessage
+    {
+        unsigned __int64 msgId;
+        unsigned __int64 ts;
+        unsigned testCaseIndex;
+    };
+    std::vector<WrittenLogMessage> writtenMessages;
+
+    unsigned testRead(bool hiddenOnly=false, bool visibleOnly=false, unsigned year=0, unsigned month=0, unsigned day=0)
+    {
+        unsigned readCount=0;
+        try
+        {
+            std::set<unsigned> matchedMessages; // used to make sure every message written has been read back
+            Owned<ISysInfoLoggerMsgIterator> iter = createSysInfoLoggerMsgIterator(visibleOnly, hiddenOnly, year, month, day, SOURCE_COMPONENT_UNITTEST);
+            ForEach(*iter)
+            {
+                const ISysInfoLoggerMsg & sysInfoMsg = iter->query();
+
+                if (strcmp(sysInfoMsg.querySource(), SOURCE_COMPONENT_UNITTEST)!=0)
+                    continue; // not a message written by this unittest so ignore
+
+                // Check written message matches read message
+                unsigned __int64 msgId = sysInfoMsg.queryLogMsgId();
+                auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [msgId] (const auto & wm){ return (wm.msgId == msgId); });
+                CPPUNIT_ASSERT_MESSAGE("Message read back not matching messages written by this unittest", matched!=writtenMessages.end());
+
+                // Make sure written messages matches message read back
+                matchedMessages.insert(matched->testCaseIndex);
+                TestCase & testCase = testCases[matched->testCaseIndex];
+                ASSERT(testCase.hidden==sysInfoMsg.queryIsHidden());
+                ASSERT(testCase.code==sysInfoMsg.queryLogMsgCode());
+                ASSERT(strcmp(testCase.msg,sysInfoMsg.queryMsg())==0);
+                ASSERT(testCase.cat.queryAudience()==sysInfoMsg.queryAudience());
+                ASSERT(testCase.cat.queryClass()==sysInfoMsg.queryClass());
+
+                readCount++;
+            }
+            ASSERT(readCount==matchedMessages.size()); // make sure there are no duplicates
+        }
+        catch (IException *e)
+        {
+            StringBuffer msg;
+            msg.appendf("testRead(hidden=%s, visible=%s) failed: ", boolToStr(hiddenOnly), boolToStr(visibleOnly));
+            e->errorMessage(msg);
+            msg.appendf("(code %d)", e->errorCode());
+            e->Release();
+            CPPUNIT_FAIL(msg.str());
+        }
+        return readCount;
+    }
+
+public:
+    ~DaliSysInfoLoggerTester()
+    {
+        daliClientEnd();
+    }
+    void testInit()
+    {
+        daliClientInit();
+    }
+    void testWrite()
+    {
+        writtenMessages.clear();
+        unsigned testCaseIndex=0;
+        for (auto testCase: testCases)
+        {
+            try
+            {
+                CDateTime dateTime;
+                dateTime.setString(testCase.dateTimeStamp);
+
+                unsigned __int64 ts = dateTime.getTimeStamp();
+                unsigned __int64 msgId = logSysInfoError(testCase.cat, testCase.code, SOURCE_COMPONENT_UNITTEST, testCase.msg, ts);
+                writtenMessages.push_back({msgId, ts, testCaseIndex++});
+                if (testCase.hidden)
+                {
+                    Owned<ISysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter(msgId, SOURCE_COMPONENT_UNITTEST);
+                    ASSERT(hideLogSysInfoMsg(msgFilter)==1);
+                }
+            }
+            catch (IException *e)
+            {
+                StringBuffer msg;
+                msg.append("logSysInfoError failed: ");
+                e->errorMessage(msg);
+                msg.appendf("(code %d)", e->errorCode());
+                e->Release();
+                CPPUNIT_FAIL(msg.str());
+            }
+        }
+        ASSERT(testCases.size()==writtenMessages.size());
+    }
+    void testSysInfoLogger()
+    {
+        // cleanup - remove messages that may have been left over from previous run
+        deleteOlderThanLogSysInfoMsg(false, false, 2001, 03, 00, SOURCE_COMPONENT_UNITTEST);
+        // Start of tests
+        testWrite();
+        ASSERT(testRead(false, false)==9);
+        ASSERT(testRead(false, false, 2000, 02, 03)==4);
+        ASSERT(testRead(false, false, 2000, 02, 04)==5);
+        ASSERT(testRead(false, true)==5); //all visible messages
+        ASSERT(testRead(true, false)==4); //all hidden messages
+        ASSERT(deleteOlderThanLogSysInfoMsg(false, true, 2000, 02, 03, SOURCE_COMPONENT_UNITTEST)==2);
+        ASSERT(deleteOlderThanLogSysInfoMsg(true, false, 2000, 02, 04, SOURCE_COMPONENT_UNITTEST)==5);
+
+        // testCase[7] and [8] are the only 2 remaining
+        // Delete single message test: delete testCase[7]
+        unsigned testCaseId = 7;
+        auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [testCaseId] (const auto & wm){ return (wm.testCaseIndex == testCaseId); });
+        if (matched==writtenMessages.end())
+            throw makeStringExceptionV(-1, "Can't find test case %u in written messages", testCaseId);
+
+        Owned<ISysInfoLoggerMsgFilter> msgFilter = createSysInfoLoggerMsgFilter(matched->msgId, SOURCE_COMPONENT_UNITTEST);
+        ASSERT(deleteLogSysInfoMsg(msgFilter)==1);
+
+        // Verify only 1 message remaining
+        ASSERT(testRead(false, false)==1);
+        // Delete 2000/02/04 and 2000/02/03 (one message but there are 2 parents remaining)
+        ASSERT(deleteOlderThanLogSysInfoMsg(false, false, 2000, 02, 05, SOURCE_COMPONENT_UNITTEST)==1);
+        // There shouldn't be any records remaining
+        ASSERT(testRead(false, false)==0);
+
+        testWrite();
+
+        // delete all messages with MsgCode 42303 -> 3 messages
+        msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST));
+        msgFilter->setMatchCode(42304);
+        ASSERT(deleteLogSysInfoMsg(msgFilter)==3);
+
+        // delete all messages matching source=SOURCE_COMPONENT_UNITTEST
+        msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST));
+        ASSERT(deleteLogSysInfoMsg(msgFilter)==6);
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( DaliSysInfoLoggerTester );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( DaliSysInfoLoggerTester, "DaliSysInfoLoggerTester" );
+
 #endif // _USE_CPPUNIT
diff --git a/testing/unittests/filetests.cpp b/testing/unittests/filetests.cpp
new file mode 100644
index 00000000000..c2c29ea718e
--- /dev/null
+++ b/testing/unittests/filetests.cpp
@@ -0,0 +1,185 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+/*
+ * File regression tests
+ *
+ */
+
+#ifdef _USE_CPPUNIT
+#include <memory>
+#include <chrono>
+#include <algorithm>
+#include "jsem.hpp"
+#include "jfile.hpp"
+#include "jdebug.hpp"
+#include "jset.hpp"
+#include "rmtfile.hpp"
+#include "jlzw.hpp"
+#include "jqueue.hpp"
+#include "jregexp.hpp"
+#include "jsecrets.hpp"
+#include "jutil.hpp"
+#include "junicode.hpp"
+
+#include "opentelemetry/sdk/common/attribute_utils.h"
+#include "opentelemetry/sdk/resource/resource.h"
+
+#include "unittests.hpp"
+
+#define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x ? x : ""),std::string(y ? y : ""))
+
+class JlibFileTest : public CppUnit::TestFixture
+{
+public:
+    CPPUNIT_TEST_SUITE(JlibFileTest);
+        CPPUNIT_TEST(testCompressed);
+        CPPUNIT_TEST(cleanup);
+    CPPUNIT_TEST_SUITE_END();
+
+    static constexpr const char * testFilename = "unittests_compressfile";
+    void createCompressed()
+    {
+        Owned<IFile> file(createIFile(testFilename));
+        Owned<ICompressedFileIO> io(createCompressedFileWriter(file, 0, false, false, nullptr));
+
+        constexpr size_t cnt = 10000;
+        constexpr size_t size = 1000;
+        offset_t pos = 0;
+        for (unsigned i = 0; i < cnt; i++)
+        {
+            byte temp[size];
+
+            for (unsigned j = 0; j < size; j += 4)
+            {
+                temp[j] = (byte)j;
+                temp[j+1] = (byte)j+1;
+                temp[j+2] = (byte)j+2;
+                temp[j+3] = (byte)random();
+            }
+
+            io->write(pos, size, temp);
+            pos += size;
+        }
+    }
+    void readCompressed(bool errorExpected)
+    {
+        bool success = false;
+        try
+        {
+            Owned<IFile> file(createIFile(testFilename));
+            Owned<ICompressedFileIO> io(createCompressedFileReader(file));
+
+            constexpr size_t cnt = 10000;
+            constexpr size_t size = 1000;
+            offset_t pos = 0;
+            for (unsigned i = 0; i < cnt; i++)
+            {
+                byte temp[size];
+
+                io->read(pos, size, temp);
+
+                for (unsigned j = 0; j < size; j += 4)
+                {
+                    CPPUNIT_ASSERT_EQUAL(temp[j], (byte)j);
+                    CPPUNIT_ASSERT_EQUAL(temp[j+1], (byte)(j+1));
+                }
+
+                pos += size;
+            }
+
+            success = true;
+        }
+        catch (IException *e)
+        {
+            if (errorExpected)
+            {
+                DBGLOG(e, "Expected error reading compressed file:");
+            }
+            else
+            {
+                StringBuffer msg("Unexpected error reading compressed file:");
+                e->errorMessage(msg);
+                CPPUNIT_FAIL(msg.str());
+            }
+            e->Release();
+        }
+        if (success && errorExpected)
+            CPPUNIT_FAIL("Expected error reading compressed file");
+    }
+    void read(offset_t offset, size32_t size, void * data)
+    {
+        Owned<IFile> file(createIFile(testFilename));
+        Owned<IFileIO> io(file->open(IFOread));
+        io->read(offset, size, data);
+    }
+    void write(offset_t offset, size32_t size, void * data)
+    {
+        Owned<IFile> file(createIFile(testFilename));
+        Owned<IFileIO> io(file->open(IFOwrite));
+        io->write(offset, size, data);
+    }
+    void testCompressed()
+    {
+        //patch the first block with zeros
+        constexpr byte zeros[0x100000] = { 0 };
+
+        createCompressed();
+        readCompressed(false);
+
+        write(0, sizeof(zeros), (void *)zeros);
+        readCompressed(true);
+
+        createCompressed();
+        write(0x10000, sizeof(zeros), (void *)zeros);
+        readCompressed(true);
+
+        createCompressed();
+        write(0x9000, sizeof(zeros), (void *)zeros);
+        readCompressed(true);
+
+        //Test the second block being corrupted with zeros
+        size32_t firstBlockSize = 0;
+        createCompressed();
+        read(4, sizeof(firstBlockSize), &firstBlockSize);
+        write(8+firstBlockSize, sizeof(zeros), (void *)zeros);
+        readCompressed(true);
+
+        //Test the data after the second block being corrupted with zeros
+        createCompressed();
+        read(4, sizeof(firstBlockSize), &firstBlockSize);
+        write(8+4+firstBlockSize, sizeof(zeros), (void *)zeros);
+        readCompressed(true);
+
+        //Test the second block being corrupted to an invalid size
+        size32_t newSize = 1;
+        createCompressed();
+        read(4, sizeof(firstBlockSize), &firstBlockSize);
+        write(8+firstBlockSize, sizeof(newSize), &newSize);
+        readCompressed(true);
+    }
+    void cleanup()
+    {
+        Owned<IFile> file(createIFile(testFilename));
+        file->remove();
+    }
+};
+
+CPPUNIT_TEST_SUITE_REGISTRATION( JlibFileTest );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibFileTest, "JlibFileTest" );
+
+#endif
diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp
index 8412fb26b62..a19ebe36159 100644
--- a/testing/unittests/jlibtests.cpp
+++ b/testing/unittests/jlibtests.cpp
@@ -3489,6 +3489,7 @@ class MachineInfoTimingTest : public CppUnit::TestFixture
                 DBGLOG(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)",
                         (unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000),
                         (unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches());
+                DBGLOG(" Throttled: Periods(%llu/%llu) TimeNs(%llu)", deltaSystem.getNumThrottledPeriods(), deltaSystem.getNumPeriods(), deltaSystem.getTimeThrottledNs());
             }
 
             for (unsigned j=0; j < i*100000000; j++)
@@ -4853,6 +4854,7 @@ class JLibStringTest : public CppUnit::TestFixture
 public:
     CPPUNIT_TEST_SUITE(JLibStringTest);
         CPPUNIT_TEST(testStristr);
+        CPPUNIT_TEST(testMemMem);
     CPPUNIT_TEST_SUITE_END();
 
     void testStristr()
@@ -4870,6 +4872,21 @@ class JLibStringTest : public CppUnit::TestFixture
         CPPUNIT_ASSERT_EQUAL_STR(stristr("", "ABC"), "");
         CPPUNIT_ASSERT_EQUAL_STR(stristr("ABC", ""), "");
     }
+
+    void testMemMem()
+    {
+        constexpr const char * haystack = "abcdefghijklmnopqrstuvwxyz";
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 0, nullptr));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(3, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(2, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+7), jmemmem(10, haystack, 3, "hij"));
+        CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(10, haystack, 3, "ijk"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 1, "i"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(8, haystack, 1, "i"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(9, haystack, 2, "ij"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 2, "ij"));
+    }
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( JLibStringTest );
diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp
index 4279f5e55d3..6b2f48964c2 100644
--- a/testing/unittests/unittests.cpp
+++ b/testing/unittests/unittests.cpp
@@ -190,6 +190,7 @@ int main(int argc, const char *argv[])
         excludeNames.append("*stress*");
         excludeNames.append("*timing*");
         excludeNames.append("*slow*");
+        excludeNames.append("Dali*"); // disabled by default as dali not available when executed by smoketest
     }
 
     if (!includeNames.length())
diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp
index 520ae4aae7a..fd2f2cfa07a 100644
--- a/thorlcr/graph/thgraph.cpp
+++ b/thorlcr/graph/thgraph.cpp
@@ -2750,6 +2750,9 @@ void CJobBase::init()
     failOnLeaks = getOptBool(THOROPT_FAIL_ON_LEAKS);
     maxLfnBlockTimeMins = getOptInt(THOROPT_MAXLFN_BLOCKTIME_MINS, DEFAULT_MAXLFN_BLOCKTIME_MINS);
     soapTraceLevel = getOptInt(THOROPT_SOAP_TRACE_LEVEL, 1);
+    StringBuffer tmpSepString;
+    getOpt(THOROPT_SOAP_LOG_SEP_STRING, tmpSepString);
+    setSoapSepString(tmpSepString.str());
 
     StringBuffer tracing("maxActivityCores = ");
     if (maxActivityCores)
diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp
index 744fbb31e31..9858848c552 100644
--- a/thorlcr/thorutil/thormisc.hpp
+++ b/thorlcr/thorutil/thormisc.hpp
@@ -118,6 +118,7 @@
 #define THOROPT_MEMORY_SPILL_AT "memorySpillAt"                                   // The threshold (%) that roxiemem will request memory to be reduced (default=80)
 #define THOROPT_FAIL_ON_LEAKS "failOnLeaks"                                       // If any leaks are detected at the end of graph, fail the query (default=false)
 #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel"                                 // The trace SOAP level (default=1)
+#define THOROPT_SOAP_LOG_SEP_STRING "soapLogSepString"                            // The SOAP request/response separator string for logging (default="")
 #define THOROPT_SORT_ALGORITHM "sortAlgorithm"                                    // The algorithm used to sort records (quicksort/mergesort)
 #define THOROPT_COMPRESS_ALLFILES "v9_4_compressAllOutputs"                       // Compress all output files (default: bare-metal=off, cloud=on)
 #define THOROPT_AVOID_RENAME "avoidRename"                                        // Avoid rename, write directly to target physical filenames (no temp file)