From 2cf2397b924492666050fdcdb41ffb6e88396583 Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Tue, 21 May 2024 12:57:52 +0100 Subject: [PATCH] HPCC-31775 Hook jlog to use api to log operator messages Signed-off-by: Shamser Ahmed --- dali/base/daclient.cpp | 4 ++ dali/base/sysinfologger.cpp | 76 ++++++++++++++++++++++ dali/base/sysinfologger.hpp | 4 +- esp/scm/ws_workunits.ecm | 2 +- esp/scm/ws_workunits_struct.ecm | 1 + esp/services/ws_workunits/ws_wudetails.cpp | 31 ++++++++- testing/unittests/dalitests.cpp | 1 + 7 files changed, 116 insertions(+), 3 deletions(-) diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp index 7a55e473dc1..6414aa0812a 100644 --- a/dali/base/daclient.cpp +++ b/dali/base/daclient.cpp @@ -29,6 +29,7 @@ #include "dautils.hpp" #include "daclient.hpp" +#include "sysinfologger.hpp" extern bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned timeout,DaliClientRole role); extern void stopClientProcess(); @@ -141,6 +142,8 @@ bool initClientProcess(IGroup *servergrp, DaliClientRole role, unsigned mpport, covengrp->Release(); queryLogMsgManager()->setSession(myProcessSession()); + if (getGlobalConfigSP()->getPropBool("@enableGlobalSysLog", true)) + UseDaliForOperatorMessages(); if (!isContainerized()) // The Environment is bare-metal only { // auto install environment monitor for server roles @@ -177,6 +180,7 @@ void removeShutdownHook(IDaliClientShutdown &shutdown) void closedownClientProcess() { + UseDaliForOperatorMessages(false); if (!daliClientIsActive) return; while (shutdownHooks.ordinality()) diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp index 3bd277bb2a1..d52caae0891 100644 --- a/dali/base/sysinfologger.cpp +++ b/dali/base/sysinfologger.cpp @@ -694,3 +694,79 @@ unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigne return count; } + +class DaliMsgLoggerHandler : public CInterfaceOf +{ +public: + DaliMsgLoggerHandler(unsigned _messageFields=MSGFIELD_all) : messageFields(_messageFields) + { + } + virtual void handleMessage(const LogMsg & msg) override + { + LogMsgSysInfo sysInfo = msg.querySysInfo(); + time_t timeNum = sysInfo.queryTime(); + unsigned __int64 ts = sysInfo.queryTime() * 1000000 + sysInfo.queryUSecs(); + logSysInfoError(msg.queryCategory(), msg.queryCode(), queryComponentName(), msg.queryText(), ts); + } + virtual bool needsPrep() const override + { + return false; + } + virtual void prep() override + { + } + virtual unsigned queryMessageFields() const override + { + return messageFields; + } + virtual void setMessageFields(unsigned _fields = MSGFIELD_all) override + { + messageFields = _fields; + } + virtual void addToPTree(IPropertyTree * parent) const override + { + IPropertyTree * handlerTree = createPTree(ipt_caseInsensitive); + handlerTree->setProp("@type", "globalmessages"); + handlerTree->setPropInt("@fields", messageFields); + parent->addPropTree("handler", handlerTree); + } + virtual int flush() override + { + return 0; + } + virtual bool getLogName(StringBuffer &name) const override + { + return false; + } + virtual offset_t getLogPosition(StringBuffer &logFileName) const override + { + return 0; + } +private: + unsigned messageFields = MSGFIELD_all; +}; + +void UseDaliForOperatorMessages(bool use) +{ + static ILogMsgHandler *msgHandler=NULL; + if (use==(msgHandler!=NULL)) + return; + if (use) + { + msgHandler = getDaliMsgLoggerHandler(); + ILogMsgFilter * operatorFilter = getCategoryLogMsgFilter(MSGAUD_operator, + MSGCLS_disaster|MSGCLS_error|MSGCLS_warning, + WarnMsgThreshold); + queryLogMsgManager()->addMonitorOwn(msgHandler, operatorFilter); + } + else + { + queryLogMsgManager()->removeMonitor(msgHandler); + msgHandler = NULL; + } +} + +ILogMsgHandler * getDaliMsgLoggerHandler() +{ + return new DaliMsgLoggerHandler(); +} diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp index 2eb0af56edd..9c7ba6c14a0 100644 --- a/dali/base/sysinfologger.hpp +++ b/dali/base/sysinfologger.hpp @@ -78,7 +78,7 @@ typedef IIteratorOf ISysInfoLoggerMsgIterator; SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr); SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr); SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr); -SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter); SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts); SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); @@ -89,4 +89,6 @@ SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilte SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source=nullptr); +SYSINFO_API ILogMsgHandler * getDaliMsgLoggerHandler(); +SYSINFO_API void UseDaliForOperatorMessages(bool use=true); #endif diff --git a/esp/scm/ws_workunits.ecm b/esp/scm/ws_workunits.ecm index 3a9b731ed57..c97b75c1291 100644 --- a/esp/scm/ws_workunits.ecm +++ b/esp/scm/ws_workunits.ecm @@ -109,7 +109,7 @@ ESPservice [ ESPmethod [cache_seconds(60), min_ver("1.57")] WUListArchiveFiles(WUListArchiveFilesRequest, WUListArchiveFilesResponse); ESPmethod [cache_seconds(60), min_ver("1.57")] WUGetArchiveFile(WUGetArchiveFileRequest, WUGetArchiveFileResponse); ESPmethod [cache_seconds(60), min_ver("1.61")] WUGetNumFileToCopy(WUGetNumFileToCopyRequest, WUGetNumFileToCopyResponse); - ESPmethod [min_ver("1.71")] WUDetails(WUDetailsRequest, WUDetailsResponse); + ESPmethod [min_ver("1.72")] WUDetails(WUDetailsRequest, WUDetailsResponse); ESPmethod [cache_seconds(600), min_ver("1.71")] WUDetailsMeta(WUDetailsMetaRequest, WUDetailsMetaResponse); ESPmethod [clear_cache_group, min_ver("1.72")] WUEclDefinitionAction(WUEclDefinitionActionRequest, WUEclDefinitionActionResponse); ESPmethod [cache_seconds(60), min_ver("1.74")] WUGetThorJobQueue(WUGetThorJobQueueRequest, WUGetThorJobQueueResponse); diff --git a/esp/scm/ws_workunits_struct.ecm b/esp/scm/ws_workunits_struct.ecm index 21883a9e4de..f7d57804c13 100644 --- a/esp/scm/ws_workunits_struct.ecm +++ b/esp/scm/ws_workunits_struct.ecm @@ -836,6 +836,7 @@ ESPstruct [nil_remove] WUResponseNote nonNegativeInteger ErrorCode; string Severity; nonNegativeInteger Cost; + [min_ver("1.72")] uint64 Id; // unique identifier for note }; ESPstruct [nil_remove] WUResponseScope diff --git a/esp/services/ws_workunits/ws_wudetails.cpp b/esp/services/ws_workunits/ws_wudetails.cpp index bd295fbab37..d513261eb38 100644 --- a/esp/services/ws_workunits/ws_wudetails.cpp +++ b/esp/services/ws_workunits/ws_wudetails.cpp @@ -22,6 +22,9 @@ #include "workunit.hpp" #include "jset.hpp" #include "jstatcodes.h" +#include "sysinfologger.hpp" + +#define COMPONENT_NOTE_MASK 0xFFFFFC0 typedef std::pair AttribValuePair; bool operator==(const AttribValuePair & p1, const AttribValuePair & p2) @@ -487,8 +490,13 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse { Owned componentConfig = getComponentConfig(); IArrayOf espWuResponseNotes; - + // Global messages + // Note: there are 2 types of 'notes': + // 1) static component warnings (usual from Helm templates) + // 2) dynamic global messages generated from executing components + // Static component messages will have COMPONENT_NOTE_MASK. Owned iter = componentConfig->getElements("warnings"); + unsigned __int64 id=1; ForEach(*iter) { IPropertyTree & cur = iter->query(); @@ -502,6 +510,27 @@ void WUDetails::processRequest(IEspWUDetailsRequest &req, IEspWUDetailsResponse espWuResponseNote->setErrorCode_null(); espWuResponseNote->setSeverity(cur.queryProp("@severity")); espWuResponseNote->setCost(0); + VStringBuffer idstr("%" I64F "u", id|COMPONENT_NOTE_MASK); + espWuResponseNote->setId(idstr.str()); + espWuResponseNotes.append(*espWuResponseNote.getClear()); + id++; + } + Owned msgFilter = createSysInfoLoggerMsgFilter(); + msgFilter->setVisibleOnly(); + Owned msgIter = createSysInfoLoggerMsgIterator(msgFilter); + ForEach(*msgIter) + { + ISysInfoLoggerMsg & sysInfoMsg = msgIter->query(); + Owned espWuResponseNote = createWUResponseNote("",""); + StringBuffer tmpbuf; + encodeXML(sysInfoMsg.queryMsg(), tmpbuf, ENCODE_NEWLINES, strlen(sysInfoMsg.queryMsg()), true); + espWuResponseNote->setSource(sysInfoMsg.querySource()); + espWuResponseNote->setMessage(tmpbuf.str()); + espWuResponseNote->setErrorCode(sysInfoMsg.queryLogMsgCode()); + espWuResponseNote->setSeverity(LogMsgClassToVarString(sysInfoMsg.queryClass())); + espWuResponseNote->setCost(0); + VStringBuffer idstr("%" I64F "u", sysInfoMsg.queryLogMsgId()); + espWuResponseNote->setId(idstr.str()); espWuResponseNotes.append(*espWuResponseNote.getClear()); } Owned respScope = createWUResponseScope("",""); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index c10a7529d2d..47493bd96d1 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -3182,6 +3182,7 @@ class DaliSysInfoLoggerTester : public CppUnit::TestFixture void testInit() { daliClientInit(); + UseDaliForOperatorMessages(false); } void testWrite() {