diff --git a/dali/base/CMakeLists.txt b/dali/base/CMakeLists.txt index 7046aae6bcb..07e7b44695a 100644 --- a/dali/base/CMakeLists.txt +++ b/dali/base/CMakeLists.txt @@ -39,7 +39,8 @@ set ( SRCS dasds.cpp dasess.cpp dasubs.cpp - dautils.cpp + dautils.cpp + sysinfologger.cpp ) set ( INCLUDES @@ -56,6 +57,7 @@ set ( INCLUDES dasess.hpp dasubs.hpp dautils.hpp + sysinfologger.hpp ) set_source_files_properties(dasds.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) @@ -70,6 +72,7 @@ include_directories ( ${HPCC_SOURCE_DIR}/rtl/include ${HPCC_SOURCE_DIR}/system/security/shared ${HPCC_SOURCE_DIR}/system/security/cryptohelper + ${HPCC_SOURCE_DIR}/testing/unittests ) ADD_DEFINITIONS( -D_USRDLL -DDALI_EXPORTS -DNULL_DALIUSER_STACKTRACE) @@ -88,5 +91,6 @@ if(NOT PLUGIN) mp hrpc dafsclient + ${CppUnit_LIBRARIES} ) endif() diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp new file mode 100644 index 00000000000..242d303733f --- /dev/null +++ b/dali/base/sysinfologger.cpp @@ -0,0 +1,696 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#include "sysinfologger.hpp" +#include "daclient.hpp" +#include "jutil.hpp" + +#define SDS_LOCK_TIMEOUT (5*60*1000) // 5 minutes +#define SYS_INFO_VERSION "1.0" + +#define SYS_INFO_ROOT "/SysLogs" +#define ATTR_NEXTID "@nextId" +#define ATTR_VERSION "@version" +#define MSG_NODE "msg" +#define ATTR_ID "@id" +#define ATTR_TIMESTAMP "@ts" +#define ATTR_AUDIENCE "@audience" +#define ATTR_CLASS "@class" +#define ATTR_CODE "@code" +#define ATTR_HIDDEN "@hidden" +#define ATTR_SOURCE "@source" + +static void extractDate(timestamp_type ts, unsigned & year, unsigned & month, unsigned & day) +{ + CDateTime timeStamp; + timeStamp.setTimeStamp(ts); + timeStamp.getDate(year, month, day); +} + +static unsigned __int64 makeMessageId(unsigned year, unsigned month, unsigned day, unsigned id) +{ + return id<<21 | year<<9 | month<<5 | day; +} + +static unsigned __int64 makeMessageId(unsigned __int64 ts, unsigned id) +{ + unsigned year, month, day; + extractDate(ts, year, month, day); + return makeMessageId(year, month, day, id); +} + +static void decodeMessageId(unsigned __int64 msgId, unsigned & year, unsigned & month, unsigned & day, unsigned & id) +{ + day = msgId & 0x1F; + month = (msgId>>5) & 0x0F; + year = (msgId>>9) & 0xFFF; + id = (msgId>>21); +} + +class CSysInfoLoggerMsg : implements ISysInfoLoggerMsg +{ + Owned msgPtree; + bool updateable = false; + + inline void ensureUpdateable() + { + if (!updateable) + throw makeStringException(-1, "Unable to update ISysInfoLoggerMsg"); + } + +public: + CSysInfoLoggerMsg() + { + msgPtree.setown(createPTree(MSG_NODE)); + } + CSysInfoLoggerMsg(unsigned id, const LogMsgCategory & cat, LogMsgCode code, const char * source, const char * msg, timestamp_type ts, bool hidden) + { + msgPtree.setown(createPTree(MSG_NODE)); + msgPtree->setPropInt64(ATTR_ID, id); + msgPtree->setPropBool(ATTR_HIDDEN, hidden); + msgPtree->setPropInt64(ATTR_TIMESTAMP, ts); + msgPtree->setPropInt(ATTR_CODE, code); + msgPtree->setProp(ATTR_AUDIENCE, LogMsgAudienceToFixString(cat.queryAudience())); + msgPtree->setProp(ATTR_CLASS, LogMsgClassToFixString(cat.queryClass())); + msgPtree->setProp(ATTR_SOURCE, source); + msgPtree->setProp(".", msg); + } + CSysInfoLoggerMsg & set(IPropertyTree * ptree, bool _updateable) + { + msgPtree.setown(ptree); + updateable = _updateable; + return * this; + } + virtual bool queryIsHidden() const override + { + return msgPtree->getPropBool(ATTR_HIDDEN, false); + } + virtual timestamp_type queryTimeStamp() const override + { + return msgPtree->getPropInt64(ATTR_TIMESTAMP); + } + virtual const char * querySource() const override + { + if (msgPtree->hasProp(ATTR_SOURCE)) + return msgPtree->queryProp(ATTR_SOURCE); + else + return "Unknown"; + } + virtual LogMsgCode queryLogMsgCode() const override + { + return msgPtree->getPropInt(ATTR_CODE, -1); + } + virtual LogMsgAudience queryAudience() const override + { + if (msgPtree->hasProp(ATTR_AUDIENCE)) + return LogMsgAudFromAbbrev(msgPtree->queryProp(ATTR_AUDIENCE)); + else + return MSGAUD_unknown; + } + virtual LogMsgClass queryClass() const override + { + if (msgPtree->hasProp(ATTR_CLASS)) + return LogMsgClassFromAbbrev(msgPtree->queryProp(ATTR_CLASS)); + else + return MSGCLS_unknown; + } + virtual unsigned __int64 queryLogMsgId() const override + { + return makeMessageId(queryTimeStamp(), msgPtree->getPropInt64(ATTR_ID, 0)); + } + virtual const char * queryMsg() const override + { + const char *msg = msgPtree->queryProp(nullptr); + return msg ? msg : ""; + } + void setHidden(bool _hidden) + { + ensureUpdateable(); + msgPtree->setPropBool(ATTR_HIDDEN, _hidden); + } + StringBuffer & getXpath(StringBuffer & xpath) + { + unsigned year, month, day; + extractDate(queryTimeStamp(), year, month, day); + unsigned __int64 id = msgPtree->getPropInt64(ATTR_ID, 0); + xpath.appendf("m%04u%02u/d%02u/" MSG_NODE "[" ATTR_ID "='%" I64F "u']", year, month, day, id); + return xpath; + } + IPropertyTree * getTree() + { + return msgPtree.getLink(); + } +}; + +class CSysInfoLoggerMsgFilter : public CSimpleInterfaceOf +{ + // (For numeric fields: match only if it has a non-zero value) + bool hiddenOnly = false; + bool visibleOnly = false; + timestamp_type matchTimeStamp = 0; + StringAttr matchSource; // only matchSource when not empty + LogMsgCode matchCode = 0; + LogMsgAudience matchAudience = MSGAUD_all; + LogMsgClass matchClass = MSGCLS_all; + bool haveDateRange = false; + unsigned matchEndYear = 0; + unsigned matchEndMonth = 0; + unsigned matchEndDay = 0; + unsigned matchStartYear = 0; + unsigned matchStartMonth = 0; + unsigned matchStartDay = 0; + unsigned matchId = 0; + +public: + CSysInfoLoggerMsgFilter(const char *_source): matchSource(_source) + { + } + CSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *_source): matchSource(_source) + { + setMatchMsgId(msgId); + } + CSysInfoLoggerMsgFilter(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *_source) : + visibleOnly(_visibleOnly), hiddenOnly(_hiddenOnly), matchSource(_source) + { + if (hiddenOnly && visibleOnly) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: cannot filter by both hiddenOnly and visibleOnly"); + setDateRange(_year, _month, _day, _year, _month, _day); + } + virtual void setHiddenOnly() override + { + hiddenOnly = true; + } + virtual void setVisibleOnly() override + { + visibleOnly = true; + } + virtual void setMatchTimeStamp(timestamp_type ts) override + { + matchTimeStamp = ts; + } + virtual void setMatchSource(const char * source) override + { + matchSource.set(source); + } + virtual void setMatchCode(LogMsgCode code) override + { + matchCode = code; + } + virtual void setMatchAudience(LogMsgAudience audience) override + { + matchAudience = audience; + } + virtual void setMatchMsgClass(LogMsgClass msgClass) override + { + matchClass = msgClass; + } + virtual void setMatchMsgId(unsigned __int64 msgId) override + { + unsigned year, month, day, id; + decodeMessageId(msgId, year, month, day, id); + if (year==0 || month==0 || day==0 || id==0) + throw makeStringExceptionV(-1,"ISysInfoLoggerMsgFilter::setMatchMsgId invalid argument: %" I64F "u", msgId); + matchEndYear = matchStartYear = year; + matchEndMonth = matchStartMonth = month; + matchEndDay = matchStartDay = day; + matchId = id; + haveDateRange = false; + } + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) override + { + if ( (startDay && (!startMonth||!startYear)) || + (endDay && (!endMonth||!endYear)) ) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: month and year must be provided when filtering by day"); + if ((!startYear && startMonth) || (!endYear && endMonth)) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: year must be provided when filtering by month"); + // Make sure starts are on or before end dates + if ( (startYear > endYear) || (startMonth && (startYear == endYear && startMonth > endMonth)) + || (startDay && (startYear == endYear && startMonth == endMonth && startDay > endDay)) ) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: invalid date range: %04u-%02u-%02u to %04u-%02u-%02u", startYear, startMonth, startDay, endYear, endMonth, endDay); + matchEndYear = endYear; + matchEndMonth = endMonth; + matchEndDay = endDay; + matchStartYear = startYear; + matchStartMonth = startMonth; + matchStartDay = startDay; + if (matchEndYear||matchStartYear) + haveDateRange = (matchStartYearmatchEndYear) + return false; + if (tyear==matchEndYear) + { + if (matchEndMonth) + { + if (tmonth>matchEndMonth) + return false; + if (tmonth==matchEndMonth) + { + if (matchEndDay && tday>matchEndDay) + return false; + } + } + } + } + return true; + } + virtual bool queryHiddenOnly() const override + { + return hiddenOnly; + } + virtual bool queryVisibleOnly() const override + { + return visibleOnly; + } + virtual timestamp_type queryMatchTimeStamp() const override + { + return matchTimeStamp; + } + virtual unsigned queryStartYear() const override + { + return matchStartYear; + } + virtual unsigned queryStartMonth() const override + { + return matchStartMonth; + } + virtual unsigned queryStartDay() const override + { + return matchStartDay; + } + virtual unsigned queryEndYear() const override + { + return matchEndYear; + } + virtual unsigned queryEndMonth() const override + { + return matchEndMonth; + } + virtual unsigned queryEndDay() const override + { + return matchEndDay; + } + virtual const char * queryMatchSource() const override + { + return matchSource.str(); + } + virtual LogMsgCode queryMatchCode() const override + { + return matchCode; + } + virtual LogMsgAudience queryMatchAudience() const override + { + return matchAudience; + } + virtual LogMsgClass queryMatchClass() const override + { + return matchClass; + } + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const override + { + bool fullDayMatch=false; + bool hardMatchYear = matchStartYear && (matchStartYear==matchEndYear); + bool hardMatchMonth = matchStartMonth && (matchStartMonth==matchEndMonth); + if (hardMatchYear && hardMatchMonth) + { + // future: optimize when month unknown with "m%04u*" + xpath.appendf("m%04u%02u", matchStartYear, matchStartMonth); + if (matchStartDay==matchEndDay) + { + xpath.appendf("/d%02u", matchStartDay); + fullDayMatch = true; + } + } + if (fullDayMatch) + xpath.appendf("/" MSG_NODE); + else + xpath.appendf("//" MSG_NODE); + if (hiddenOnly) + xpath.append("[" ATTR_HIDDEN "='1')]"); + if (visibleOnly) + xpath.append("[" ATTR_HIDDEN "='0')]"); + if (!matchSource.isEmpty()) + xpath.appendf("[" ATTR_SOURCE "='%s']", matchSource.str()); + if (matchCode) + xpath.appendf("[" ATTR_CODE "='%d']", (int)matchCode); + if (matchAudience!=MSGAUD_all) + xpath.appendf("[" ATTR_AUDIENCE "='%s']", LogMsgAudienceToFixString(matchAudience)); + if (matchClass!=MSGCLS_all) + xpath.appendf("[" ATTR_CLASS "='%s']", LogMsgClassToFixString(matchClass)); + if (matchId) + xpath.appendf("[" ATTR_ID "='%u']", matchId); + if (matchTimeStamp) + xpath.appendf("[" ATTR_TIMESTAMP "='%" I64F "u']", matchTimeStamp); + return xpath; + } +}; + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source) +{ + return new CSysInfoLoggerMsgFilter(source); +} + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source) +{ + return new CSysInfoLoggerMsgFilter(msgId, source); +} + +class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOf +{ + Linked filter; + // N.b. IRemoteConnection exists for the duration of the iterator so if this iterator exists for too long, it could cause + // performance issues for other clients: consider caching some messages and releasing connection (and reopening as necessary). + Owned conn; + bool updateable = false; + Owned msgIter; + CSysInfoLoggerMsg infoMsg; + + bool ensureMatch() + { + if (filter->hasDateRange()) + { + for (; msgIter->isValid(); msgIter->next()) + { + timestamp_type ts = msgIter->query().getPropInt64(ATTR_TIMESTAMP, 0); + if (filter->isInDateRange(ts)) + return true; + } + return false; + } + return msgIter->isValid(); + } + +public: + CSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * _filter, bool _updateable=false, IRemoteConnection *_conn=nullptr) : filter(_filter), updateable(_updateable), conn(_conn) + { + if (!conn) + { + unsigned mode = updateable ? RTM_LOCK_WRITE : RTM_LOCK_READ; + conn.setown(querySDS().connect(SYS_INFO_ROOT, myProcessSession(), mode, SDS_LOCK_TIMEOUT)); + if (!conn) + throw makeStringExceptionV(-1, "CSysInfoLoggerMsgIterator: unable to create connection to '%s'", SYS_INFO_ROOT); + } + } + CSysInfoLoggerMsg & queryInfoLoggerMsg() + { + return infoMsg.set(&(msgIter->get()), updateable); + } + virtual ISysInfoLoggerMsg & query() override + { + return queryInfoLoggerMsg(); + } + virtual bool first() override + { + StringBuffer xpath; + filter->getQualifierXPathFilter(xpath); + + msgIter.setown(conn->queryRoot()->getElements(xpath.str())); + if (!msgIter->first()) + return false; + return ensureMatch(); + } + virtual bool next() override + { + if (msgIter->next()) + return false; + return ensureMatch(); + } + virtual bool isValid() override + { + return msgIter ? msgIter->isValid() : false; + } +}; + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source) +{ + Owned filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day, source); + return new CSysInfoLoggerMsgIterator(filter, false); +} + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return new CSysInfoLoggerMsgIterator(msgFilter, false); +} + +// returns messageId +unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, timestamp_type ts) +{ + if (ts==0) + ts = getTimeStampNowValue(); + + if (isEmptyString(source)) + source = "unknown"; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT); + if (!conn) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", SYS_INFO_ROOT); + + IPropertyTree * root = conn->queryRoot(); + unsigned id = root->getPropInt(ATTR_NEXTID, 1); + if (id==UINT_MAX) // wrap id to reuse id numbers (shouldn't wrap but no harm in doing this for safety) + id=1; + root->setPropInt(ATTR_NEXTID, id+1); + + StringBuffer xpath; + unsigned year, month, day; + extractDate(ts, year, month, day); + xpath.appendf("%s/m%04u%02u/d%02u/%s", SYS_INFO_ROOT, year, month, day, MSG_NODE); + Owned connMsgRoot = querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE_ADD, SDS_LOCK_TIMEOUT); + if (!connMsgRoot) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", xpath.str()); + IPropertyTree * msgPT = connMsgRoot->queryRoot(); + + CSysInfoLoggerMsg sysInfoMsg(id, cat, code, source, msg, ts, false); + msgPT->setPropTree(nullptr, sysInfoMsg.getTree()); + msgPT->setProp(".", msg); // previous setPropTree doesn't set the node value + return makeMessageId(ts, id); +} + +unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function updateOp) +{ + unsigned count = 0; + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, true); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + updateOp(sysInfoMsg); + ++count; + } + return count; +} + +unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function updateOp) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, source); + return updateMessage(msgFilter, updateOp); +} + +unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);}); +} + +bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1; +} + +unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);}); +} + +bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1; +} + +unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + std::vector deleteXpathList; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + { + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, false, conn.getLink()); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + StringBuffer xpath; + sysInfoMsg.getXpath(xpath); + deleteXpathList.push_back(xpath.str()); + } + } + IPropertyTree * root = conn->queryRoot(); + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())) + ++count; + } + return count; +} + +bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, source); + return deleteLogSysInfoMsg(msgFilter); +} + +unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source) +{ + if (!year && month) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: year must be provided if month is specified (year=%u, month=%u, day=%u)", year, month, day); + if (!month && day) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: month must be provided if day is specified (year=%u, month=%u, day=%u)", year, month, day); + if (month>12) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid month(year=%u, month=%u, day=%u)", year, month, day); + if (day>31) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid day(year=%u, month=%u, day=%u)", year, month, day); + // With visibleOnly/hiddenOnly option, use createSysInfoLoggerMsgFilter() + if (visibleOnly || hiddenOnly || day) + { + unsigned count = 0; + Owned msgFilter = createSysInfoLoggerMsgFilter(source); + if (hiddenOnly) + msgFilter->setHiddenOnly(); + if (visibleOnly) + msgFilter->setVisibleOnly(); + if (source) + msgFilter->setMatchSource(source); + msgFilter->setOlderThanDate(year, month, day); + return deleteLogSysInfoMsg(msgFilter); + } + + // With only date range, use this quicker method to remove whole subtrees + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + if (!conn) + return 0; + + std::vector deleteXpathList; + IPropertyTree * root = conn->queryRoot(); + // future: optimize by getting only minimum set of subtrees to delete and get sorted elements(so search can stop earlier) + Owned monthIter = root->getElements("*"); + Owned innerException; //only first exception record/reported + ForEach(*monthIter) + { + IPropertyTree & monthPT = monthIter->query(); + if (year==0) + deleteXpathList.push_back(monthPT.queryName()); + else + { + unsigned msgYear = 0, msgMonth = 0; + const char *p = monthPT.queryName(); // should be in format 'myyyydd' + if (*p++ == 'm') + { + msgYear = readDigits(p, 4, false); + msgMonth = readDigits(p, 2, false); + } + if (msgYear == 0 || msgMonth == 0) + { + if (!innerException) + innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT " is invalid: %s", monthPT.queryName())); + continue; + } + if (msgYear > year) + continue; + if (msgYear < year) + deleteXpathList.push_back(monthPT.queryName()); + else + { + // msgYear matches year in this section + if (msgMonth > month) + continue; + else if (msgMonth < month) + deleteXpathList.push_back(monthPT.queryName()); + else // msgMonth==month + { + Owned dayIter = monthPT.getElements("*"); + ForEach(*dayIter) + { + IPropertyTree & dayPT = dayIter->query(); + unsigned msgDay = 0; + const char * d = dayPT.queryName(); + if (*d++ == 'd') + msgDay = readDigits(d, 2); + if (msgDay == 0) + { + if (!innerException) + innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT "/%s is invalid: %s", monthPT.queryName(), dayPT.queryName())); + continue; + } + if (day && (msgDay >= day)) + continue; + + VStringBuffer xpath("%s/%s", monthPT.queryName(), dayPT.queryName()); + deleteXpathList.push_back(xpath.str()); + } + } + } + } + } + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())); + ++count; + } + + if (innerException) // allow items to be deleted even if there is an exception + throw innerException.getClear(); + + return count; +} diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp new file mode 100644 index 00000000000..2eb0af56edd --- /dev/null +++ b/dali/base/sysinfologger.hpp @@ -0,0 +1,92 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#ifndef SYSINFOLOGGER +#define SYSINFOLOGGER + +#include "jlog.hpp" +#include "jutil.hpp" + +#ifdef DALI_EXPORT + #define SYSINFO_API DECL_EXPORT +#else + #define SYSINFO_API DECL_IMPORT +#endif + +interface ISysInfoLoggerMsg +{ + virtual bool queryIsHidden() const = 0; + virtual timestamp_type queryTimeStamp() const = 0; + virtual const char * querySource() const = 0; + virtual LogMsgCode queryLogMsgCode() const = 0; + virtual LogMsgAudience queryAudience() const = 0; + virtual LogMsgClass queryClass() const = 0; + virtual unsigned __int64 queryLogMsgId() const = 0; + virtual const char * queryMsg() const = 0; +}; + +interface IConstSysInfoLoggerMsgFilter : public IInterface +{ + virtual bool hasDateRange() const = 0; + virtual bool isInDateRange(timestamp_type ts) const = 0; + virtual bool queryHiddenOnly() const = 0; + virtual bool queryVisibleOnly() const = 0; + virtual timestamp_type queryMatchTimeStamp() const = 0; + virtual unsigned queryStartYear() const = 0; + virtual unsigned queryStartMonth() const = 0; + virtual unsigned queryStartDay() const = 0; + virtual unsigned queryEndYear() const = 0; + virtual unsigned queryEndMonth() const = 0; + virtual unsigned queryEndDay() const = 0; + virtual const char * queryMatchSource() const = 0; + virtual LogMsgCode queryMatchCode() const = 0; + virtual LogMsgAudience queryMatchAudience() const = 0; + virtual LogMsgClass queryMatchClass() const = 0; + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const = 0; +}; + +interface ISysInfoLoggerMsgFilter : extends IConstSysInfoLoggerMsgFilter +{ + virtual void setHiddenOnly() = 0; + virtual void setVisibleOnly() = 0; + virtual void setMatchTimeStamp(timestamp_type ts) = 0; + virtual void setMatchSource(const char * source) = 0; + virtual void setMatchCode(LogMsgCode code) = 0; + virtual void setMatchAudience(LogMsgAudience audience) = 0; + virtual void setMatchMsgClass(LogMsgClass msgClass) = 0; + virtual void setMatchMsgId(unsigned __int64 msgId) = 0; + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) = 0; + virtual void setOlderThanDate(unsigned year, unsigned month, unsigned day) = 0; +}; + +typedef IIteratorOf ISysInfoLoggerMsgIterator; + +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter); + +SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts); +SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter, const char *source=nullptr); +SYSINFO_API bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source=nullptr); + +#endif diff --git a/system/jlib/jtime.cpp b/system/jlib/jtime.cpp index 87ed2384b7b..58dca7f0639 100644 --- a/system/jlib/jtime.cpp +++ b/system/jlib/jtime.cpp @@ -117,19 +117,6 @@ time_t timelocal(struct tm * local) #endif //__GNUC__ -static unsigned readDigits(char const * & str, unsigned numDigits) -{ - unsigned ret = 0; - while(numDigits--) - { - char c = *str++; - if(!isdigit(c)) - throwError1(JLIBERR_BadlyFormedDateTime, str); - ret = ret * 10 + (c - '0'); - } - return ret; -} - static void checkChar(char const * & str, char required) { char c = *str++; @@ -222,22 +209,32 @@ void CDateTime::set(time_t simple) void CDateTime::setString(char const * str, char const * * end, bool local) { + char const * beginstr = str; // save for error message if (!str||!*str) { clear(); return; } - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); - checkChar(str, 'T'); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); - unsigned nano = 0; + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0, nano = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + checkChar(str, 'T'); + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } + if(*str == '.') { unsigned digits; @@ -256,26 +253,44 @@ void CDateTime::setString(char const * str, char const * * end, bool local) void CDateTime::setDateString(char const * str, char const * * end) { - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } if(end) *end = str; set(year, month, day, 0, 0, 0, 0, false); } void CDateTime::setTimeString(char const * str, char const * * end, bool local) { - unsigned year; - unsigned month; - unsigned day; + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0; getDate(year, month, day, false); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); + + try + { + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } unsigned nano = 0; if(*str == '.') { @@ -668,10 +683,18 @@ void CScmDateTime::setString(const char * pstr) else if ((sign == '-') || (sign == '+')) { end++; - int delta = readDigits(end, 2); - if (*end++ != ':') + int delta = 0; + try + { + delta = readDigits(end, 2); + checkChar(end, ':'); + delta = delta * 60 + readDigits(end, 2); + } + catch (IException * e) + { + e->Release(); throwError1(JLIBERR_BadlyFormedDateTime, pstr); - delta = delta * 60 + readDigits(end, 2); + } if (sign == '-') delta = -delta; utcToLocalDelta = delta; diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp index e2013d8399e..d22fda4888d 100644 --- a/system/jlib/jutil.cpp +++ b/system/jlib/jutil.cpp @@ -3632,3 +3632,20 @@ void hold(const char *msg) } + +unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure) +{ + unsigned ret = 0; + while (numDigits--) + { + char c = *str++; + if (!isdigit(c)) + { + if (throwOnFailure) + throw makeStringExceptionV(-1, "Invalid format (readDigits): %s", str); + return 0; + } + ret = ret * 10 + (c - '0'); + } + return ret; +} diff --git a/system/jlib/jutil.hpp b/system/jlib/jutil.hpp index 07c81dbc2cd..4841d61761b 100644 --- a/system/jlib/jutil.hpp +++ b/system/jlib/jutil.hpp @@ -655,6 +655,8 @@ struct HPCCBuildInfo extern jlib_decl HPCCBuildInfo hpccBuildInfo; extern jlib_decl bool checkCreateDaemon(unsigned argc, const char * * argv); +extern jlib_decl unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure = true); + //Createpassword of specified length, containing UpperCaseAlphas, LowercaseAlphas, numerics and symbols extern jlib_decl const char * generatePassword(StringBuffer &pwd, int pwdLen); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index 042546a3995..cbba197fbf4 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -37,6 +37,7 @@ #include #include "unittests.hpp" +#include "sysinfologger.hpp" //#define COMPAT @@ -3034,4 +3035,231 @@ class CFileNameNormalizeUnitTest : public CppUnit::TestFixture, CDfsLogicalFileN CPPUNIT_TEST_SUITE_REGISTRATION( CFileNameNormalizeUnitTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CFileNameNormalizeUnitTest, "CFileNameNormalizeUnitTest" ); +#define SOURCE_COMPONENT_UNITTEST "sysinfologger-unittest" + +class DaliSysInfoLoggerTester : public CppUnit::TestFixture +{ + /* Note: global messages will be written for dates between 2000-02-04 and 2000-02-05 */ + /* Note: All global messages with time stamp before 2000-03-31 will be deleted */ + CPPUNIT_TEST_SUITE(DaliSysInfoLoggerTester); + CPPUNIT_TEST(testInit); + CPPUNIT_TEST(testSysInfoLogger); + CPPUNIT_TEST_SUITE_END(); + + struct TestCase + { + LogMsgCategory cat; + LogMsgCode code; + bool hidden; + const char * dateTimeStamp; + const char * msg; + }; + + std::vector testCases = + { + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-03T10:01:22.342343", + "CSysInfoLogger Unit test message 1" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-03T12:03:42.114233", + "CSysInfoLogger Unit test message 2" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + true, + "2000-02-03T14:02:13.678443", + "CSysInfoLogger Unit test message 3" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-03T16:05:18.8324832", + "CSysInfoLogger Unit test message 4" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-04T03:01:42.5754", + "CSysInfoLogger Unit test message 5" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-04T09:06:25.133132", + "CSysInfoLogger Unit test message 6" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + false, + "2000-02-04T11:09:32.78439", + "CSysInfoLogger Unit test message 7" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T13:02:12.82821", + "CSysInfoLogger Unit test message 8" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T18:32:11.23421", + "CSysInfoLogger Unit test message 9" + } + }; + + struct WrittenLogMessage + { + unsigned __int64 msgId; + unsigned __int64 ts; + unsigned testCaseIndex; + }; + std::vector writtenMessages; + + unsigned testRead(bool hiddenOnly=false, bool visibleOnly=false, unsigned year=0, unsigned month=0, unsigned day=0) + { + unsigned readCount=0; + try + { + std::set matchedMessages; // used to make sure every message written has been read back + Owned iter = createSysInfoLoggerMsgIterator(visibleOnly, hiddenOnly, year, month, day, SOURCE_COMPONENT_UNITTEST); + ForEach(*iter) + { + const ISysInfoLoggerMsg & sysInfoMsg = iter->query(); + + if (strcmp(sysInfoMsg.querySource(), SOURCE_COMPONENT_UNITTEST)!=0) + continue; // not a message written by this unittest so ignore + + // Check written message matches read message + unsigned __int64 msgId = sysInfoMsg.queryLogMsgId(); + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [msgId] (const auto & wm){ return (wm.msgId == msgId); }); + CPPUNIT_ASSERT_MESSAGE("Message read back not matching messages written by this unittest", matched!=writtenMessages.end()); + + // Make sure written messages matches message read back + matchedMessages.insert(matched->testCaseIndex); + TestCase & testCase = testCases[matched->testCaseIndex]; + ASSERT(testCase.hidden==sysInfoMsg.queryIsHidden()); + ASSERT(testCase.code==sysInfoMsg.queryLogMsgCode()); + ASSERT(strcmp(testCase.msg,sysInfoMsg.queryMsg())==0); + ASSERT(testCase.cat.queryAudience()==sysInfoMsg.queryAudience()); + ASSERT(testCase.cat.queryClass()==sysInfoMsg.queryClass()); + + readCount++; + } + ASSERT(readCount==matchedMessages.size()); // make sure there are no duplicates + } + catch (IException *e) + { + StringBuffer msg; + msg.appendf("testRead(hidden=%s, visible=%s) failed: ", boolToStr(hiddenOnly), boolToStr(visibleOnly)); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + return readCount; + } + +public: + ~DaliSysInfoLoggerTester() + { + daliClientEnd(); + } + void testInit() + { + daliClientInit(); + } + void testWrite() + { + writtenMessages.clear(); + unsigned testCaseIndex=0; + for (auto testCase: testCases) + { + try + { + CDateTime dateTime; + dateTime.setString(testCase.dateTimeStamp); + + unsigned __int64 ts = dateTime.getTimeStamp(); + unsigned __int64 msgId = logSysInfoError(testCase.cat, testCase.code, SOURCE_COMPONENT_UNITTEST, testCase.msg, ts); + writtenMessages.push_back({msgId, ts, testCaseIndex++}); + if (testCase.hidden) + { + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, SOURCE_COMPONENT_UNITTEST); + ASSERT(hideLogSysInfoMsg(msgFilter)==1); + } + } + catch (IException *e) + { + StringBuffer msg; + msg.append("logSysInfoError failed: "); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + } + ASSERT(testCases.size()==writtenMessages.size()); + } + void testSysInfoLogger() + { + // cleanup - remove messages that may have been left over from previous run + deleteOlderThanLogSysInfoMsg(false, false, 2001, 03, 00, SOURCE_COMPONENT_UNITTEST); + // Start of tests + testWrite(); + ASSERT(testRead(false, false)==9); + ASSERT(testRead(false, false, 2000, 02, 03)==4); + ASSERT(testRead(false, false, 2000, 02, 04)==5); + ASSERT(testRead(false, true)==5); //all visible messages + ASSERT(testRead(true, false)==4); //all hidden messages + ASSERT(deleteOlderThanLogSysInfoMsg(false, true, 2000, 02, 03, SOURCE_COMPONENT_UNITTEST)==2); + ASSERT(deleteOlderThanLogSysInfoMsg(true, false, 2000, 02, 04, SOURCE_COMPONENT_UNITTEST)==5); + + // testCase[7] and [8] are the only 2 remaining + // Delete single message test: delete testCase[7] + unsigned testCaseId = 7; + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [testCaseId] (const auto & wm){ return (wm.testCaseIndex == testCaseId); }); + if (matched==writtenMessages.end()) + throw makeStringExceptionV(-1, "Can't find test case %u in written messages", testCaseId); + + Owned msgFilter = createSysInfoLoggerMsgFilter(matched->msgId, SOURCE_COMPONENT_UNITTEST); + ASSERT(deleteLogSysInfoMsg(msgFilter)==1); + + // Verify only 1 message remaining + ASSERT(testRead(false, false)==1); + // Delete 2000/02/04 and 2000/02/03 (one message but there are 2 parents remaining) + ASSERT(deleteOlderThanLogSysInfoMsg(false, false, 2000, 02, 05, SOURCE_COMPONENT_UNITTEST)==1); + // There shouldn't be any records remaining + ASSERT(testRead(false, false)==0); + + testWrite(); + + // delete all messages with MsgCode 42303 -> 3 messages + msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST)); + msgFilter->setMatchCode(42304); + ASSERT(deleteLogSysInfoMsg(msgFilter)==3); + + // delete all messages matching source=SOURCE_COMPONENT_UNITTEST + msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST)); + ASSERT(deleteLogSysInfoMsg(msgFilter)==6); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( DaliSysInfoLoggerTester ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( DaliSysInfoLoggerTester, "DaliSysInfoLoggerTester" ); + #endif // _USE_CPPUNIT diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index c11154dfe2c..dbe78e92f55 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -190,6 +190,7 @@ int main(int argc, const char *argv[]) excludeNames.append("*stress*"); excludeNames.append("*timing*"); excludeNames.append("*slow*"); + excludeNames.append("Dali*"); // disabled by default as dali not available when executed by smoketest } if (!includeNames.length())