From 2185c5c423d94c7bdb60fe16f762c6fc03cfb0ba Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Fri, 10 May 2024 17:18:16 +0100 Subject: [PATCH] HPCC-31774 Implement API logging, hiding, deleting and querying global messages This provides an API for loggging, hiding, deleting messages and query using Dali as the message store. The API has not been integrated into any existing software, so existing software is not affected. Unittests have been implemented for API, named DaliSysInfoLoggerTester. Signed-off-by: Shamser Ahmed --- dali/base/CMakeLists.txt | 6 +- dali/base/sysinfologger.cpp | 682 ++++++++++++++++++++++++++++++++ dali/base/sysinfologger.hpp | 92 +++++ system/jlib/jtime.cpp | 105 +++-- system/jlib/jutil.cpp | 17 + system/jlib/jutil.hpp | 2 + testing/unittests/dalitests.cpp | 229 +++++++++++ testing/unittests/unittests.cpp | 1 + 8 files changed, 1092 insertions(+), 42 deletions(-) create mode 100644 dali/base/sysinfologger.cpp create mode 100644 dali/base/sysinfologger.hpp diff --git a/dali/base/CMakeLists.txt b/dali/base/CMakeLists.txt index 7046aae6bcb..07e7b44695a 100644 --- a/dali/base/CMakeLists.txt +++ b/dali/base/CMakeLists.txt @@ -39,7 +39,8 @@ set ( SRCS dasds.cpp dasess.cpp dasubs.cpp - dautils.cpp + dautils.cpp + sysinfologger.cpp ) set ( INCLUDES @@ -56,6 +57,7 @@ set ( INCLUDES dasess.hpp dasubs.hpp dautils.hpp + sysinfologger.hpp ) set_source_files_properties(dasds.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) @@ -70,6 +72,7 @@ include_directories ( ${HPCC_SOURCE_DIR}/rtl/include ${HPCC_SOURCE_DIR}/system/security/shared ${HPCC_SOURCE_DIR}/system/security/cryptohelper + ${HPCC_SOURCE_DIR}/testing/unittests ) ADD_DEFINITIONS( -D_USRDLL -DDALI_EXPORTS -DNULL_DALIUSER_STACKTRACE) @@ -88,5 +91,6 @@ if(NOT PLUGIN) mp hrpc dafsclient + ${CppUnit_LIBRARIES} ) endif() diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp new file mode 100644 index 00000000000..e0245fcf171 --- /dev/null +++ b/dali/base/sysinfologger.cpp @@ -0,0 +1,682 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#include "sysinfologger.hpp" +#include "daclient.hpp" +#include "jutil.hpp" + +#define SDS_LOCK_TIMEOUT (5*60*1000) // 5 minutes +#define SYS_INFO_VERSION "1.0" + +#define SYS_INFO_ROOT "/SysLogs" +#define ATTR_NEXTID "@nextId" +#define ATTR_VERSION "@version" +#define MSG_NODE "msg" +#define ATTR_ID "@id" +#define ATTR_TIMESTAMP "@ts" +#define ATTR_AUDIENCE "@audience" +#define ATTR_CLASS "@class" +#define ATTR_CODE "@code" +#define ATTR_HIDDEN "@hidden" +#define ATTR_SOURCE "@source" + +static void extractDate(unsigned __int64 ts, unsigned & year, unsigned & month, unsigned & day) +{ + CDateTime timeStamp; + timeStamp.setTimeStamp(ts); + timeStamp.getDate(year, month, day); +} + +static unsigned __int64 makeMessageId(unsigned year, unsigned month, unsigned day, unsigned id) +{ + return id<<21 | year<<9 | month<<5 | day; +} + +static unsigned __int64 makeMessageId(unsigned __int64 ts, unsigned id) +{ + unsigned year, month, day; + extractDate(ts, year, month, day); + return makeMessageId(year, month, day, id); +} + +static void decodeMessageId(unsigned __int64 msgId, unsigned & year, unsigned & month, unsigned & day, unsigned & id) +{ + day = msgId & 0x1F; + month = (msgId>>5) & 0x0F; + year = (msgId>>9) & 0xFFF; + id = (msgId>>21); +} + +class CSysInfoLoggerMsg : implements ISysInfoLoggerMsg +{ + Owned msgPtree; + bool updateable = false; + + inline void ensureUpdateable() + { + if (!updateable) + throw makeStringException(-1, "Unable to update ISysInfoLoggerMsg"); + } + +public: + CSysInfoLoggerMsg() + { + msgPtree.setown(createPTree(MSG_NODE)); + } + CSysInfoLoggerMsg(unsigned id, const LogMsgCategory & cat, LogMsgCode code, const char * source, const char * msg, unsigned __int64 ts, bool hidden) + { + msgPtree.setown(createPTree(MSG_NODE)); + msgPtree->setPropInt64(ATTR_ID, id); + msgPtree->setPropBool(ATTR_HIDDEN, hidden); + msgPtree->setPropInt64(ATTR_TIMESTAMP, ts); + msgPtree->setPropInt(ATTR_CODE, code); + msgPtree->setProp(ATTR_AUDIENCE, LogMsgAudienceToFixString(cat.queryAudience())); + msgPtree->setProp(ATTR_CLASS, LogMsgClassToFixString(cat.queryClass())); + msgPtree->setProp(ATTR_SOURCE, source); + msgPtree->setProp(".", msg); + } + CSysInfoLoggerMsg & set(IPropertyTree * ptree, bool _updateable) + { + msgPtree.setown(ptree); + updateable = _updateable; + return * this; + } + virtual bool queryIsHidden() const override + { + return msgPtree->getPropBool(ATTR_HIDDEN, false); + } + virtual unsigned __int64 queryTimeStamp() const override + { + return msgPtree->getPropInt64(ATTR_TIMESTAMP); + } + virtual const char * querySource() const override + { + if (msgPtree->hasProp(ATTR_SOURCE)) + return msgPtree->queryProp(ATTR_SOURCE); + else + return "Unknown"; + } + virtual LogMsgCode queryLogMsgCode() const override + { + return msgPtree->getPropInt(ATTR_CODE, -1); + } + virtual LogMsgAudience queryAudience() const override + { + if (msgPtree->hasProp(ATTR_AUDIENCE)) + return LogMsgAudFromAbbrev(msgPtree->queryProp(ATTR_AUDIENCE)); + else + return MSGAUD_unknown; + } + virtual LogMsgClass queryClass() const override + { + if (msgPtree->hasProp(ATTR_CLASS)) + return LogMsgClassFromAbbrev(msgPtree->queryProp(ATTR_CLASS)); + else + return MSGCLS_unknown; + } + virtual unsigned __int64 queryLogMsgId() const override + { + return makeMessageId(queryTimeStamp(), msgPtree->getPropInt64(ATTR_ID, 0)); + } + virtual const char * queryMsg() const override + { + const char *msg = msgPtree->queryProp(nullptr); + return msg ? msg : ""; + } + void setHidden(bool _hidden) + { + ensureUpdateable(); + msgPtree->setPropBool(ATTR_HIDDEN, _hidden); + } + StringBuffer & getXpath(StringBuffer & xpath) + { + unsigned year, month, day; + extractDate(queryTimeStamp(), year, month, day); + unsigned __int64 id = msgPtree->getPropInt64(ATTR_ID, 0); + xpath.appendf("m%04u%02u/d%02u/" MSG_NODE "[" ATTR_ID "='%" I64F "u']", year, month, day, id); + return xpath; + } + IPropertyTree * getTree() + { + return msgPtree.getLink(); + } +}; + +class CSysInfoLoggerMsgFilter : public CSimpleInterfaceOf +{ + // (For numeric fields: match only if it has a non-zero value) + bool hiddenOnly = false; + bool visibleOnly = false; + unsigned __int64 matchTimeStamp = 0; + StringAttr matchSource; // only matchSource when not empty + LogMsgCode matchCode = 0; + LogMsgAudience matchAudience = MSGAUD_all; + LogMsgClass matchClass = MSGCLS_all; + bool haveDateRange = false; + unsigned matchEndYear = 0; + unsigned matchEndMonth = 0; + unsigned matchEndDay = 0; + unsigned matchStartYear = 0; + unsigned matchStartMonth = 0; + unsigned matchStartDay = 0; + unsigned matchId = 0; + +public: + CSysInfoLoggerMsgFilter() + { + } + CSysInfoLoggerMsgFilter(unsigned __int64 msgId) + { + setMatchMsgId(msgId); + } + CSysInfoLoggerMsgFilter(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day) : + visibleOnly(_visibleOnly), hiddenOnly(_hiddenOnly), + matchEndYear(_year), matchEndMonth(_month), matchEndDay(_day), + matchStartYear(_year), matchStartMonth(_month), matchStartDay(_day) + { + if (hiddenOnly && visibleOnly) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: cannot filter by both hiddenOnly and visibleOnly"); + setDateRange(_year, _month, _day, _year, _month, _day); + } + virtual void setHiddenOnly() override + { + hiddenOnly = true; + } + virtual void setVisibleOnly() override + { + visibleOnly = true; + } + virtual void setMatchTimeStamp(unsigned __int64 ts) override + { + matchTimeStamp = ts; + } + virtual void setMatchSource(const char * source) override + { + matchSource.set(source); + } + virtual void setMatchCode(LogMsgCode code) override + { + matchCode = code; + } + virtual void setMatchAudience(LogMsgAudience audience) override + { + matchAudience = audience; + } + virtual void setMatchMsgClass(LogMsgClass msgClass) override + { + matchClass = msgClass; + } + virtual void setMatchMsgId(unsigned __int64 msgId) override + { + unsigned year, month, day, id; + decodeMessageId(msgId, year, month, day, id); + if (year==0 || month==0 || day==0 || id==0) + throw makeStringExceptionV(-1,"ISysInfoLoggerMsgFilter::setMatchMsgId invalid argument: %" I64F "u", msgId); + matchEndYear = matchStartYear = year; + matchEndMonth = matchStartMonth = month; + matchEndDay = matchStartDay = day; + matchId = id; + haveDateRange = false; + } + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) override + { + if ( (startDay && (!startMonth||!startYear)) || + (endDay && (!endMonth||!endYear)) ) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: month and year must be provided when filtering by day"); + if ((!startYear && startMonth) || (!endYear && endMonth)) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: year must be provided when filtering by month"); + // Make sure starts are on or before end dates + if ( (startYear > endYear) || (startMonth && (startYear == endYear && startMonth > endMonth)) + || (startDay && (startYear == endYear && startMonth == endMonth && startDay > endDay)) ) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: invalid date range: %04u-%02u-%02u to %04u-%02u-%02u", startYear, startMonth, startDay, endYear, endMonth, endDay); + matchEndYear = endYear; + matchEndMonth = endMonth; + matchEndDay = endDay; + matchStartYear = startYear; + matchStartMonth = startMonth; + matchStartDay = startDay; + if (matchEndYear||matchStartYear) + haveDateRange = (matchStartYearmatchEndYear) + return false; + if (tyear==matchEndYear) + { + if (matchEndMonth) + { + if (tmonth>matchEndMonth) + return false; + if (tmonth==matchEndMonth) + { + if (matchEndDay && tday>matchEndDay) + return false; + } + } + } + } + return true; + } + virtual bool queryHiddenOnly() const override + { + return hiddenOnly; + } + virtual bool queryVisibleOnly() const override + { + return visibleOnly; + } + virtual unsigned __int64 queryMatchTimeStamp() const override + { + return matchTimeStamp; + } + virtual unsigned queryStartYear() const override + { + return matchStartYear; + } + virtual unsigned queryStartMonth() const override + { + return matchStartMonth; + } + virtual unsigned queryStartDay() const override + { + return matchStartDay; + } + virtual unsigned queryEndYear() const override + { + return matchEndYear; + } + virtual unsigned queryEndMonth() const override + { + return matchEndMonth; + } + virtual unsigned queryEndDay() const override + { + return matchEndDay; + } + virtual const char * queryMatchSource() const override + { + return matchSource.str(); + } + virtual LogMsgCode queryMatchCode() const override + { + return matchCode; + } + virtual LogMsgAudience queryMatchAudience() const override + { + return matchAudience; + } + virtual LogMsgClass queryMatchClass() const override + { + return matchClass; + } + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const override + { + bool fullDayMatch=false; + bool hardMatchYear = matchStartYear && (matchStartYear==matchEndYear); + bool hardMatchMonth = matchStartMonth && (matchStartMonth==matchEndMonth); + if (hardMatchYear && hardMatchMonth) + { + xpath.appendf("m%04u%02u", matchStartYear, matchStartMonth); + if (matchStartDay==matchEndDay) + { + xpath.appendf("/d%02u", matchStartDay); + fullDayMatch = true; + } + } + if (fullDayMatch) + xpath.appendf("/" MSG_NODE); + else + xpath.appendf("//" MSG_NODE); + if (hiddenOnly) + xpath.append("[" ATTR_HIDDEN "='1')]"); + if (visibleOnly) + xpath.append("[" ATTR_HIDDEN "='0')]"); + if (!matchSource.isEmpty()) + xpath.appendf("[" ATTR_SOURCE "='%s']", matchSource.str()); + if (matchCode) + xpath.appendf("[" ATTR_CODE "='%d']", (int)matchCode); + if (matchAudience!=MSGAUD_all) + xpath.appendf("[" ATTR_AUDIENCE "='%s']", LogMsgAudienceToFixString(matchAudience)); + if (matchClass!=MSGCLS_all) + xpath.appendf("[" ATTR_CLASS "='%s']", LogMsgClassToFixString(matchClass)); + if (matchId) + xpath.appendf("[" ATTR_ID "='%u']", matchId); + if (matchTimeStamp) + xpath.appendf("[" ATTR_TIMESTAMP "='%" I64F "u']", matchTimeStamp); + return xpath; + } +}; + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter() +{ + return new CSysInfoLoggerMsgFilter(); +} + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId) +{ + return new CSysInfoLoggerMsgFilter(msgId); +} + +class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOf +{ + Linked filter; + Owned conn; + bool updateable = false; + Owned msgIter; + CSysInfoLoggerMsg infoMsg; + + bool ensureMatch() + { + if (filter->hasDateRange()) + { + for (; msgIter->isValid(); msgIter->next()) + { + unsigned __int64 ts = msgIter->query().getPropInt64(ATTR_TIMESTAMP, 0); + if (filter->isInDateRange(ts)) + return true; + } + return false; + } + return msgIter->isValid(); + } + +public: + CSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * _filter, bool _updateable=false, IRemoteConnection *_conn=nullptr) : filter(_filter), updateable(_updateable), conn(_conn) + { + if (!conn) + { + unsigned mode = updateable ? RTM_LOCK_WRITE : RTM_LOCK_READ; + conn.setown(querySDS().connect(SYS_INFO_ROOT, myProcessSession(), mode, SDS_LOCK_TIMEOUT)); + if (!conn) + throw makeStringExceptionV(-1, "CSysInfoLoggerMsgIterator: unable to create connection to '%s'", SYS_INFO_ROOT); + } + } + CSysInfoLoggerMsg & queryInfoLoggerMsg() + { + return infoMsg.set(&(msgIter->get()), updateable); + } + virtual ISysInfoLoggerMsg & query() override + { + return queryInfoLoggerMsg(); + } + virtual bool first() override + { + StringBuffer xpath; + filter->getQualifierXPathFilter(xpath); + + msgIter.setown(conn->queryRoot()->getElements(xpath.str())); + if (!msgIter->first()) + return false; + return ensureMatch(); + } + virtual bool next() override + { + if (msgIter->next()) + return false; + return ensureMatch(); + } + virtual bool isValid() override + { + return msgIter ? msgIter->isValid() : false; + } +}; + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day) +{ + Owned filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day); + return new CSysInfoLoggerMsgIterator(filter, false); +} + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return new CSysInfoLoggerMsgIterator(msgFilter, false); +} + +// returns messageId +unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts) +{ + if (ts==0) + ts = getTimeStampNowValue(); + + if (isEmptyString(source)) + source = "unknown"; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT); + if (!conn) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", SYS_INFO_ROOT); + + IPropertyTree * root = conn->queryRoot(); + unsigned id = root->getPropInt(ATTR_NEXTID, 1); + if (id==UINT_MAX) // wrap id to reuse id numbers (shouldn't wrap but no harm in doing this for safety) + id=1; + root->setPropInt(ATTR_NEXTID, id+1); + + StringBuffer xpath; + unsigned year, month, day; + extractDate(ts, year, month, day); + xpath.appendf("%s/m%04u%02u/d%02u/%s", SYS_INFO_ROOT, year, month, day, MSG_NODE); + Owned connMsgRoot = querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE_ADD, SDS_LOCK_TIMEOUT); + if (!connMsgRoot) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", xpath.str()); + IPropertyTree * msgPT = connMsgRoot->queryRoot(); + + CSysInfoLoggerMsg sysInfoMsg(id, cat, code, source, msg, ts, false); + msgPT->setPropTree(nullptr, sysInfoMsg.getTree()); + msgPT->setProp(".", msg); // previous setPropTree doesn't set the node value + return makeMessageId(ts, id); +} + +unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function updateOp) +{ + unsigned count = 0; + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, true); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + updateOp(sysInfoMsg); + ++count; + } + return count; +} + +unsigned updateMessage(unsigned __int64 msgId, std::function updateOp) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId); + return updateMessage(msgFilter, updateOp); +} + +unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);}); +} + +bool hideLogSysInfoMsg(unsigned __int64 msgId) +{ + return updateMessage(msgId, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1; +} + +unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);}); +} + +bool unhideLogSysInfoMsg(unsigned __int64 msgId) +{ + return updateMessage(msgId, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1; +} + +unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + std::vector deleteXpathList; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + { + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, false, conn.getLink()); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + StringBuffer xpath; + sysInfoMsg.getXpath(xpath); + deleteXpathList.push_back(xpath.str()); + } + } + IPropertyTree * root = conn->queryRoot(); + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())); + ++count; + } + return count; +} + +bool deleteLogSysInfoMsg(unsigned __int64 msgId) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(); + msgFilter->setMatchMsgId(msgId); + return deleteLogSysInfoMsg(msgFilter); +} + +unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day) +{ + if (!year && month) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: year must be provided if month is specified (year=%u, month=%u, day=%u)", year, month, day); + if (!month && day) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: month must be provided if day is specified (year=%u, month=%u, day=%u)", year, month, day); + if (month>12) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid month(year=%u, month=%u, day=%u)", year, month, day); + if (day>31) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid day(year=%u, month=%u, day=%u)", year, month, day); + // With visibleOnly/hiddenOnly option, use createSysInfoLoggerMsgFilter() + if (visibleOnly || hiddenOnly) + { + unsigned count = 0; + Owned msgFilter = createSysInfoLoggerMsgFilter(); + if (hiddenOnly) + msgFilter->setHiddenOnly(); + if (visibleOnly) + msgFilter->setVisibleOnly(); + msgFilter->setOlderThanDate(year, month, day); + return deleteLogSysInfoMsg(msgFilter); + } + + // With only date range, use this quicker method to remove whole subtrees + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + if (!conn) + return 0; + + std::vector deleteXpathList; + IPropertyTree * root = conn->queryRoot(); + Owned monthIter = root->getElements("*"); + ForEach(*monthIter) + { + IPropertyTree & monthPT = monthIter->query(); + if (year==0) + deleteXpathList.push_back(monthPT.queryName()); + else + { + unsigned msgYear = 0, msgMonth = 0; + const char *p = monthPT.queryName(); // should be in format 'myyyydd' + if (*p++ == 'm') + { + msgYear = readDigits(p, 4, false); + msgMonth = readDigits(p, 2, false); + } + if (msgYear == 0 || msgMonth == 0) + throw makeStringExceptionV(-1, "child of " SYS_INFO_ROOT " is invalid: %s", monthPT.queryName()); + if (msgYear > year) + continue; + if (msgYear < year) + deleteXpathList.push_back(monthPT.queryName()); + else + { + // msgYear matches year in this section + if (msgMonth > month) + continue; + else if (msgMonth < month) + deleteXpathList.push_back(monthPT.queryName()); + else // msgMonth==month + { + Owned dayIter = monthPT.getElements("*"); + ForEach(*dayIter) + { + IPropertyTree & dayPT = dayIter->query(); + unsigned msgDay = 0; + const char * d = dayPT.queryName(); + if (*d++ == 'd') + msgDay = readDigits(d, 2); + if (msgDay == 0) + throw makeStringExceptionV(-1, "child of " SYS_INFO_ROOT "/%s is invalid: %s", monthPT.queryName(), dayPT.queryName()); + if (day && (msgDay >= day)) + continue; + + VStringBuffer xpath("%s/%s", monthPT.queryName(), dayPT.queryName()); + deleteXpathList.push_back(xpath.str()); + } + } + } + } + } + + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())); + ++count; + } + + return count; +} diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp new file mode 100644 index 00000000000..488923bdfed --- /dev/null +++ b/dali/base/sysinfologger.hpp @@ -0,0 +1,92 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#ifndef SYSINFOLOGGER +#define SYSINFOLOGGER + +#include "jlog.hpp" +#include "jutil.hpp" + +#ifdef DALI_EXPORT + #define SYSINFO_API DECL_EXPORT +#else + #define SYSINFO_API DECL_IMPORT +#endif + +interface ISysInfoLoggerMsg +{ + virtual bool queryIsHidden() const = 0; + virtual unsigned __int64 queryTimeStamp() const = 0; + virtual const char * querySource() const = 0; + virtual LogMsgCode queryLogMsgCode() const = 0; + virtual LogMsgAudience queryAudience() const = 0; + virtual LogMsgClass queryClass() const = 0; + virtual unsigned __int64 queryLogMsgId() const = 0; + virtual const char * queryMsg() const = 0; +}; + +interface IConstSysInfoLoggerMsgFilter : public IInterface +{ + virtual bool hasDateRange() const = 0; + virtual bool isInDateRange(unsigned __int64 ts) const = 0; + virtual bool queryHiddenOnly() const = 0; + virtual bool queryVisibleOnly() const = 0; + virtual unsigned __int64 queryMatchTimeStamp() const = 0; + virtual unsigned queryStartYear() const = 0; + virtual unsigned queryStartMonth() const = 0; + virtual unsigned queryStartDay() const = 0; + virtual unsigned queryEndYear() const = 0; + virtual unsigned queryEndMonth() const = 0; + virtual unsigned queryEndDay() const = 0; + virtual const char * queryMatchSource() const = 0; + virtual LogMsgCode queryMatchCode() const = 0; + virtual LogMsgAudience queryMatchAudience() const = 0; + virtual LogMsgClass queryMatchClass() const = 0; + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const = 0; +}; + +interface ISysInfoLoggerMsgFilter : extends IConstSysInfoLoggerMsgFilter +{ + virtual void setHiddenOnly() = 0; + virtual void setVisibleOnly() = 0; + virtual void setMatchTimeStamp(unsigned __int64 ts) = 0; + virtual void setMatchSource(const char * source) = 0; + virtual void setMatchCode(LogMsgCode code) = 0; + virtual void setMatchAudience(LogMsgAudience audience) = 0; + virtual void setMatchMsgClass(LogMsgClass msgClass) = 0; + virtual void setMatchMsgId(unsigned __int64 msgId) = 0; + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) = 0; + virtual void setOlderThanDate(unsigned year, unsigned month, unsigned day) = 0; +}; + +typedef IIteratorOf ISysInfoLoggerMsgIterator; + +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(); +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter); + +SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts); +SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool hideLogSysInfoMsg(unsigned __int64 msgId); +SYSINFO_API unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool unhideLogSysInfoMsg(unsigned __int64 msgId); +SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId); +SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day); + +#endif diff --git a/system/jlib/jtime.cpp b/system/jlib/jtime.cpp index 87ed2384b7b..58dca7f0639 100644 --- a/system/jlib/jtime.cpp +++ b/system/jlib/jtime.cpp @@ -117,19 +117,6 @@ time_t timelocal(struct tm * local) #endif //__GNUC__ -static unsigned readDigits(char const * & str, unsigned numDigits) -{ - unsigned ret = 0; - while(numDigits--) - { - char c = *str++; - if(!isdigit(c)) - throwError1(JLIBERR_BadlyFormedDateTime, str); - ret = ret * 10 + (c - '0'); - } - return ret; -} - static void checkChar(char const * & str, char required) { char c = *str++; @@ -222,22 +209,32 @@ void CDateTime::set(time_t simple) void CDateTime::setString(char const * str, char const * * end, bool local) { + char const * beginstr = str; // save for error message if (!str||!*str) { clear(); return; } - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); - checkChar(str, 'T'); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); - unsigned nano = 0; + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0, nano = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + checkChar(str, 'T'); + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } + if(*str == '.') { unsigned digits; @@ -256,26 +253,44 @@ void CDateTime::setString(char const * str, char const * * end, bool local) void CDateTime::setDateString(char const * str, char const * * end) { - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } if(end) *end = str; set(year, month, day, 0, 0, 0, 0, false); } void CDateTime::setTimeString(char const * str, char const * * end, bool local) { - unsigned year; - unsigned month; - unsigned day; + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0; getDate(year, month, day, false); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); + + try + { + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } unsigned nano = 0; if(*str == '.') { @@ -668,10 +683,18 @@ void CScmDateTime::setString(const char * pstr) else if ((sign == '-') || (sign == '+')) { end++; - int delta = readDigits(end, 2); - if (*end++ != ':') + int delta = 0; + try + { + delta = readDigits(end, 2); + checkChar(end, ':'); + delta = delta * 60 + readDigits(end, 2); + } + catch (IException * e) + { + e->Release(); throwError1(JLIBERR_BadlyFormedDateTime, pstr); - delta = delta * 60 + readDigits(end, 2); + } if (sign == '-') delta = -delta; utcToLocalDelta = delta; diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp index e2013d8399e..d22fda4888d 100644 --- a/system/jlib/jutil.cpp +++ b/system/jlib/jutil.cpp @@ -3632,3 +3632,20 @@ void hold(const char *msg) } + +unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure) +{ + unsigned ret = 0; + while (numDigits--) + { + char c = *str++; + if (!isdigit(c)) + { + if (throwOnFailure) + throw makeStringExceptionV(-1, "Invalid format (readDigits): %s", str); + return 0; + } + ret = ret * 10 + (c - '0'); + } + return ret; +} diff --git a/system/jlib/jutil.hpp b/system/jlib/jutil.hpp index 07c81dbc2cd..4841d61761b 100644 --- a/system/jlib/jutil.hpp +++ b/system/jlib/jutil.hpp @@ -655,6 +655,8 @@ struct HPCCBuildInfo extern jlib_decl HPCCBuildInfo hpccBuildInfo; extern jlib_decl bool checkCreateDaemon(unsigned argc, const char * * argv); +extern jlib_decl unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure = true); + //Createpassword of specified length, containing UpperCaseAlphas, LowercaseAlphas, numerics and symbols extern jlib_decl const char * generatePassword(StringBuffer &pwd, int pwdLen); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index 042546a3995..870b5998df3 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -37,6 +37,7 @@ #include #include "unittests.hpp" +#include "sysinfologger.hpp" //#define COMPAT @@ -3034,4 +3035,232 @@ class CFileNameNormalizeUnitTest : public CppUnit::TestFixture, CDfsLogicalFileN CPPUNIT_TEST_SUITE_REGISTRATION( CFileNameNormalizeUnitTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CFileNameNormalizeUnitTest, "CFileNameNormalizeUnitTest" ); +#define SOURCE_COMPONENT "sysinfologger-unittest" + +class DaliSysInfoLoggerTester : public CppUnit::TestFixture +{ + /* Note: global messages will be written for dates between 2000-02-04 and 2000-02-05 */ + /* Note: All global messages with time stamp before 2000-03-31 will be deleted */ + CPPUNIT_TEST_SUITE(DaliSysInfoLoggerTester); + CPPUNIT_TEST(testInit); + CPPUNIT_TEST(testSysInfoLogger); + CPPUNIT_TEST_SUITE_END(); + + struct TestCase + { + LogMsgCategory cat; + LogMsgCode code; + bool hidden; + const char * dateTimeStamp; + const char * msg; + }; + + std::vector testCases = + { + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-03T10:01:22.342343", + "CSysInfoLogger Unit test message 1" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-03T12:03:42.114233", + "CSysInfoLogger Unit test message 2" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + true, + "2000-02-03T14:02:13.678443", + "CSysInfoLogger Unit test message 3" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-03T16:05:18.8324832", + "CSysInfoLogger Unit test message 4" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-04T03:01:42.5754", + "CSysInfoLogger Unit test message 5" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-04T09:06:25.133132", + "CSysInfoLogger Unit test message 6" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + false, + "2000-02-04T11:09:32.78439", + "CSysInfoLogger Unit test message 7" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T13:02:12.82821", + "CSysInfoLogger Unit test message 8" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T18:32:11.23421", + "CSysInfoLogger Unit test message 9" + } + }; + + struct WrittenLogMessage + { + unsigned __int64 msgId; + unsigned __int64 ts; + unsigned testCaseIndex; + }; + std::vector writtenMessages; + + unsigned testRead(bool hiddenOnly=false, bool visibleOnly=false, unsigned year=0, unsigned month=0, unsigned day=0) + { + unsigned readCount=0; + try + { + std::set matchedMessages; // used to make sure every message written has been read back + Owned iter = createSysInfoLoggerMsgIterator(visibleOnly, hiddenOnly, year, month, day); + ForEach(*iter) + { + const ISysInfoLoggerMsg & sysInfoMsg = iter->query(); + + if (strcmp(sysInfoMsg.querySource(), SOURCE_COMPONENT)!=0) + continue; // not a message written by this unittest so ignore + + // Check written message matches read message + unsigned __int64 msgId = sysInfoMsg.queryLogMsgId(); + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [msgId] (const auto & wm){ return (wm.msgId == msgId); }); + CPPUNIT_ASSERT_MESSAGE("Message read back not matching messages written by this unittest", matched!=writtenMessages.end()); + + // Make sure written messages matches message read back + matchedMessages.insert(matched->testCaseIndex); + TestCase & testCase = testCases[matched->testCaseIndex]; + ASSERT(testCase.hidden==sysInfoMsg.queryIsHidden()); + ASSERT(testCase.code==sysInfoMsg.queryLogMsgCode()); + ASSERT(strcmp(testCase.msg,sysInfoMsg.queryMsg())==0); + ASSERT(testCase.cat.queryAudience()==sysInfoMsg.queryAudience()); + ASSERT(testCase.cat.queryClass()==sysInfoMsg.queryClass()); + + readCount++; + } + ASSERT(readCount==matchedMessages.size()); // make sure there are no duplicates + } + catch (IException *e) + { + StringBuffer msg; + msg.appendf("testRead(hidden=%s, visible=%s) failed: ", boolToStr(hiddenOnly), boolToStr(visibleOnly)); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + return readCount; + } + +public: + ~DaliSysInfoLoggerTester() + { + daliClientEnd(); + } + void testInit() + { + daliClientInit(); + } + void testWrite() + { + writtenMessages.clear(); + unsigned testCaseIndex=0; + for (auto testCase: testCases) + { + try + { + CDateTime dateTime; + dateTime.setString(testCase.dateTimeStamp); + + unsigned __int64 ts = dateTime.getTimeStamp(); + unsigned __int64 msgId = logSysInfoError(testCase.cat, testCase.code, SOURCE_COMPONENT, testCase.msg, ts); + writtenMessages.push_back({msgId, ts, testCaseIndex++}); + if (testCase.hidden) + { + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId); + ASSERT(hideLogSysInfoMsg(msgFilter)==1); + } + } + catch (IException *e) + { + StringBuffer msg; + msg.append("logSysInfoError failed: "); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + } + ASSERT(testCases.size()==writtenMessages.size()); + } + void testSysInfoLogger() + { + // cleanup - remove messages that may have been left over from previous run + deleteOlderThanLogSysInfoMsg(false, false, 2001, 03, 00); + // Start of tests + testWrite(); + ASSERT(testRead(false, false)==9); + ASSERT(testRead(false, false, 2000, 02, 03)==4); + ASSERT(testRead(false, false, 2000, 02, 04)==5); + ASSERT(testRead(false, true)==5); //all visible messages + ASSERT(testRead(true, false)==4); //all hidden messages + ASSERT(deleteOlderThanLogSysInfoMsg(false, true, 2000, 02, 03)==2); + ASSERT(deleteOlderThanLogSysInfoMsg(true, false, 2000, 02, 04)==5); + + // testCase[7] and [8] are the only 2 remaining + // Delete single message test: delete testCase[7] + unsigned testCaseId = 7; + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [testCaseId] (const auto & wm){ return (wm.testCaseIndex == testCaseId); }); + if (matched==writtenMessages.end()) + throw makeStringExceptionV(-1, "Can't find test case %u in written messages", testCaseId); + + Owned msgFilter = createSysInfoLoggerMsgFilter(matched->msgId); + ASSERT(deleteLogSysInfoMsg(msgFilter)==1); + + // Verify only 1 message remaining + ASSERT(testRead(false, false)==1); + // Delete 2000/02/04 and 2000/02/03 (one message but there are 2 parents remaining) + ASSERT(deleteOlderThanLogSysInfoMsg(false, false, 2000, 02, 05)==2); + // There shouldn't be any records remaining + ASSERT(testRead(false, false)==0); + + testWrite(); + + // delete all messages with MsgCode 42303 -> 3 messages + msgFilter.setown(createSysInfoLoggerMsgFilter()); + msgFilter->setMatchCode(42304); + ASSERT(deleteLogSysInfoMsg(msgFilter)==3); + + // delete all messages matching source=SOURCE_COMPONENT + msgFilter.setown(createSysInfoLoggerMsgFilter()); + msgFilter->setMatchSource(SOURCE_COMPONENT); + ASSERT(deleteLogSysInfoMsg(msgFilter)==6); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( DaliSysInfoLoggerTester ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( DaliSysInfoLoggerTester, "DaliSysInfoLoggerTester" ); + #endif // _USE_CPPUNIT diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index c11154dfe2c..dbe78e92f55 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -190,6 +190,7 @@ int main(int argc, const char *argv[]) excludeNames.append("*stress*"); excludeNames.append("*timing*"); excludeNames.append("*slow*"); + excludeNames.append("Dali*"); // disabled by default as dali not available when executed by smoketest } if (!includeNames.length())