From 9e1219c1872553eb163d170d33e22f4658b80909 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 24 Sep 2024 12:01:38 -0400 Subject: [PATCH 01/10] HPCC-32693 ECL Watch v5 add log filter fields to ZAP dialog fixes an issue where the ECL Watch v5 UI had not been updated to show the fields necessary for WsWorkunits to retrieve relevant log files to be included in a ZAP report Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/WUDetailsWidget.js | 29 +++++++++++++++ esp/src/eclwatch/css/hpcc.css | 4 ++ .../eclwatch/templates/WUDetailsWidget.html | 37 ++++++++++++++++++- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/esp/src/eclwatch/WUDetailsWidget.js b/esp/src/eclwatch/WUDetailsWidget.js index c7d2c331af0..411bba32c52 100644 --- a/esp/src/eclwatch/WUDetailsWidget.js +++ b/esp/src/eclwatch/WUDetailsWidget.js @@ -117,6 +117,16 @@ define([ this.emailFrom = registry.byId(this.id + "EmailFrom"); this.emailSubject = registry.byId(this.id + "EmailSubject"); this.emailBody = registry.byId(this.id + "EmailBody"); + + //Zap LogFilters + this.logFilterStartDateTime = dom.byId(this.id + "StartDateTime"); + this.logFilterStartDate = registry.byId(this.id + "StartDate"); + this.logFilterStartTime = registry.byId(this.id + "StartTime"); + this.logFilterEndDateTime = dom.byId(this.id + "EndDateTime"); + this.logFilterEndDate = registry.byId(this.id + "EndDate"); + this.logFilterEndTime = registry.byId(this.id + "EndTime"); + this.logFilterRelativeTimeRangeBuffer = registry.byId(this.id + "RelativeTimeRangeBuffer"); + this.protected = registry.byId(this.id + "Protected"); this.infoGridWidget = registry.byId(this.id + "InfoContainer"); this.zapDialog = registry.byId(this.id + "ZapDialog"); @@ -146,14 +156,33 @@ define([ this.checkThorLogStatus(); }, + formatLogFilterDateTime: function (dateField, timeField, dateTimeField) { + if (dateField.value.toString() !== "Invalid Date") { + const d = new Date(dateField.value); + const date = `${d.getFullYear()}-${(d.getMonth() < 9 ? "0" : "") + parseInt(d.getMonth() + 1, 10)}-${d.getDate()}`; + const time = timeField.value.toString().replace(/.*1970\s(\S+).*/, "$1"); + dateTimeField.value = `${date}T${time}.000Z`; + } + }, + _onSubmitDialog: function () { var context = this; var includeSlaveLogsCheckbox = this.includeSlaveLogsCheckbox.get("checked"); + if (this.logFilterRelativeTimeRangeBuffer.value !== "") { + this.logFilterEndDate.required = ""; + this.logFilterStartDate.required = ""; + } if (this.zapForm.validate()) { //WUCreateAndDownloadZAPInfo is not a webservice so relying on form to submit. //Server treats "on" and '' as the same thing. this.includeSlaveLogsCheckbox.set("value", includeSlaveLogsCheckbox ? "on" : "off"); + + // Log Filters + this.formatLogFilterDateTime(this.logFilterStartDate, this.logFilterStartTime, this.logFilterStartDateTime); + this.formatLogFilterDateTime(this.logFilterEndDate, this.logFilterEndTime, this.logFilterEndDateTime); + this.zapForm.set("action", "/WsWorkunits/WUCreateAndDownloadZAPInfo"); + this.zapDialog.hide(); this.checkThorLogStatus(); if (this.logAccessorMessage !== "") { diff --git a/esp/src/eclwatch/css/hpcc.css b/esp/src/eclwatch/css/hpcc.css index 26559935861..e6d0e6f6f0b 100644 --- a/esp/src/eclwatch/css/hpcc.css +++ b/esp/src/eclwatch/css/hpcc.css @@ -74,6 +74,10 @@ form li label { padding-top: 4px; } +.dijitDialogPaneContent { + overflow-x: hidden !important; +} + .dijitDialogPaneContent form li label { float: left; width: 25%; diff --git a/esp/src/eclwatch/templates/WUDetailsWidget.html b/esp/src/eclwatch/templates/WUDetailsWidget.html index a56d36ec15c..ef58752e1a6 100644 --- a/esp/src/eclwatch/templates/WUDetailsWidget.html +++ b/esp/src/eclwatch/templates/WUDetailsWidget.html @@ -183,7 +183,7 @@

-
+
@@ -199,8 +199,41 @@

+ + + + + + + + + + + + + +

-
+
From 726f398ef0e8d7d32c4b563b0101db0f90bd341e Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 3 Oct 2024 17:44:57 +0100 Subject: [PATCH 02/10] Split off 9.2.128 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 1bcc94fea52..55b71a45818 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.127-closedown0 +version: 9.2.129-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.127-closedown0 +appVersion: 9.2.129-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e6c559d0b6e..0dd032a0298 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1361,7 +1361,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.2.127-closedown0 +helmVersion: 9.2.129-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 210fd162a1c..e39e59c893a 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 127 ) +set ( HPCC_POINT 129 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-09-27T11:17:43Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-10-03T16:44:57Z" ) ### From 11afe9f8c77c4e127ee8c15aa625296a5bce1a4b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 3 Oct 2024 17:46:09 +0100 Subject: [PATCH 03/10] Split off 9.4.102 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- version.cmake | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 4897d30687c..36d2e25e73d 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.101-closedown0 +version: 9.4.103-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.101-closedown0 +appVersion: 9.4.103-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 44cb04487ab..7676de2ad68 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1473,7 +1473,7 @@ Pass in dict with .root, .visibility defined {{- end -}} {{- define "hpcc.generateHelmVersion" -}} -helmVersion: 9.4.101-closedown0 +helmVersion: 9.4.103-closedown0 {{- end -}} {{/* diff --git a/version.cmake b/version.cmake index 61d3394d558..c050f7b2826 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 101 ) +set ( HPCC_POINT 103 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-09-27T11:20:37Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-10-03T16:46:08Z" ) ### From 689232be4376e1b7fc435fc8b73dbb6e2a4d773e Mon Sep 17 00:00:00 2001 From: Shamser Ahmed Date: Fri, 10 May 2024 17:18:16 +0100 Subject: [PATCH 04/10] HPCC-31774 Implement API logging, hiding, deleting and querying global messages This provides an API for loggging, hiding, deleting messages and query using Dali as the message store. The API has not been integrated into any existing software, so existing software is not affected. Unittests have been implemented for API, named DaliSysInfoLoggerTester. Signed-off-by: Shamser Ahmed Changes following review Signed-off-by: Shamser Ahmed --- dali/base/CMakeLists.txt | 6 +- dali/base/sysinfologger.cpp | 696 ++++++++++++++++++++++++++++++++ dali/base/sysinfologger.hpp | 92 +++++ system/jlib/jtime.cpp | 105 +++-- system/jlib/jutil.cpp | 17 + system/jlib/jutil.hpp | 2 + testing/unittests/dalitests.cpp | 228 +++++++++++ testing/unittests/unittests.cpp | 1 + 8 files changed, 1105 insertions(+), 42 deletions(-) create mode 100644 dali/base/sysinfologger.cpp create mode 100644 dali/base/sysinfologger.hpp diff --git a/dali/base/CMakeLists.txt b/dali/base/CMakeLists.txt index 7046aae6bcb..07e7b44695a 100644 --- a/dali/base/CMakeLists.txt +++ b/dali/base/CMakeLists.txt @@ -39,7 +39,8 @@ set ( SRCS dasds.cpp dasess.cpp dasubs.cpp - dautils.cpp + dautils.cpp + sysinfologger.cpp ) set ( INCLUDES @@ -56,6 +57,7 @@ set ( INCLUDES dasess.hpp dasubs.hpp dautils.hpp + sysinfologger.hpp ) set_source_files_properties(dasds.cpp PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) @@ -70,6 +72,7 @@ include_directories ( ${HPCC_SOURCE_DIR}/rtl/include ${HPCC_SOURCE_DIR}/system/security/shared ${HPCC_SOURCE_DIR}/system/security/cryptohelper + ${HPCC_SOURCE_DIR}/testing/unittests ) ADD_DEFINITIONS( -D_USRDLL -DDALI_EXPORTS -DNULL_DALIUSER_STACKTRACE) @@ -88,5 +91,6 @@ if(NOT PLUGIN) mp hrpc dafsclient + ${CppUnit_LIBRARIES} ) endif() diff --git a/dali/base/sysinfologger.cpp b/dali/base/sysinfologger.cpp new file mode 100644 index 00000000000..242d303733f --- /dev/null +++ b/dali/base/sysinfologger.cpp @@ -0,0 +1,696 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#include "sysinfologger.hpp" +#include "daclient.hpp" +#include "jutil.hpp" + +#define SDS_LOCK_TIMEOUT (5*60*1000) // 5 minutes +#define SYS_INFO_VERSION "1.0" + +#define SYS_INFO_ROOT "/SysLogs" +#define ATTR_NEXTID "@nextId" +#define ATTR_VERSION "@version" +#define MSG_NODE "msg" +#define ATTR_ID "@id" +#define ATTR_TIMESTAMP "@ts" +#define ATTR_AUDIENCE "@audience" +#define ATTR_CLASS "@class" +#define ATTR_CODE "@code" +#define ATTR_HIDDEN "@hidden" +#define ATTR_SOURCE "@source" + +static void extractDate(timestamp_type ts, unsigned & year, unsigned & month, unsigned & day) +{ + CDateTime timeStamp; + timeStamp.setTimeStamp(ts); + timeStamp.getDate(year, month, day); +} + +static unsigned __int64 makeMessageId(unsigned year, unsigned month, unsigned day, unsigned id) +{ + return id<<21 | year<<9 | month<<5 | day; +} + +static unsigned __int64 makeMessageId(unsigned __int64 ts, unsigned id) +{ + unsigned year, month, day; + extractDate(ts, year, month, day); + return makeMessageId(year, month, day, id); +} + +static void decodeMessageId(unsigned __int64 msgId, unsigned & year, unsigned & month, unsigned & day, unsigned & id) +{ + day = msgId & 0x1F; + month = (msgId>>5) & 0x0F; + year = (msgId>>9) & 0xFFF; + id = (msgId>>21); +} + +class CSysInfoLoggerMsg : implements ISysInfoLoggerMsg +{ + Owned msgPtree; + bool updateable = false; + + inline void ensureUpdateable() + { + if (!updateable) + throw makeStringException(-1, "Unable to update ISysInfoLoggerMsg"); + } + +public: + CSysInfoLoggerMsg() + { + msgPtree.setown(createPTree(MSG_NODE)); + } + CSysInfoLoggerMsg(unsigned id, const LogMsgCategory & cat, LogMsgCode code, const char * source, const char * msg, timestamp_type ts, bool hidden) + { + msgPtree.setown(createPTree(MSG_NODE)); + msgPtree->setPropInt64(ATTR_ID, id); + msgPtree->setPropBool(ATTR_HIDDEN, hidden); + msgPtree->setPropInt64(ATTR_TIMESTAMP, ts); + msgPtree->setPropInt(ATTR_CODE, code); + msgPtree->setProp(ATTR_AUDIENCE, LogMsgAudienceToFixString(cat.queryAudience())); + msgPtree->setProp(ATTR_CLASS, LogMsgClassToFixString(cat.queryClass())); + msgPtree->setProp(ATTR_SOURCE, source); + msgPtree->setProp(".", msg); + } + CSysInfoLoggerMsg & set(IPropertyTree * ptree, bool _updateable) + { + msgPtree.setown(ptree); + updateable = _updateable; + return * this; + } + virtual bool queryIsHidden() const override + { + return msgPtree->getPropBool(ATTR_HIDDEN, false); + } + virtual timestamp_type queryTimeStamp() const override + { + return msgPtree->getPropInt64(ATTR_TIMESTAMP); + } + virtual const char * querySource() const override + { + if (msgPtree->hasProp(ATTR_SOURCE)) + return msgPtree->queryProp(ATTR_SOURCE); + else + return "Unknown"; + } + virtual LogMsgCode queryLogMsgCode() const override + { + return msgPtree->getPropInt(ATTR_CODE, -1); + } + virtual LogMsgAudience queryAudience() const override + { + if (msgPtree->hasProp(ATTR_AUDIENCE)) + return LogMsgAudFromAbbrev(msgPtree->queryProp(ATTR_AUDIENCE)); + else + return MSGAUD_unknown; + } + virtual LogMsgClass queryClass() const override + { + if (msgPtree->hasProp(ATTR_CLASS)) + return LogMsgClassFromAbbrev(msgPtree->queryProp(ATTR_CLASS)); + else + return MSGCLS_unknown; + } + virtual unsigned __int64 queryLogMsgId() const override + { + return makeMessageId(queryTimeStamp(), msgPtree->getPropInt64(ATTR_ID, 0)); + } + virtual const char * queryMsg() const override + { + const char *msg = msgPtree->queryProp(nullptr); + return msg ? msg : ""; + } + void setHidden(bool _hidden) + { + ensureUpdateable(); + msgPtree->setPropBool(ATTR_HIDDEN, _hidden); + } + StringBuffer & getXpath(StringBuffer & xpath) + { + unsigned year, month, day; + extractDate(queryTimeStamp(), year, month, day); + unsigned __int64 id = msgPtree->getPropInt64(ATTR_ID, 0); + xpath.appendf("m%04u%02u/d%02u/" MSG_NODE "[" ATTR_ID "='%" I64F "u']", year, month, day, id); + return xpath; + } + IPropertyTree * getTree() + { + return msgPtree.getLink(); + } +}; + +class CSysInfoLoggerMsgFilter : public CSimpleInterfaceOf +{ + // (For numeric fields: match only if it has a non-zero value) + bool hiddenOnly = false; + bool visibleOnly = false; + timestamp_type matchTimeStamp = 0; + StringAttr matchSource; // only matchSource when not empty + LogMsgCode matchCode = 0; + LogMsgAudience matchAudience = MSGAUD_all; + LogMsgClass matchClass = MSGCLS_all; + bool haveDateRange = false; + unsigned matchEndYear = 0; + unsigned matchEndMonth = 0; + unsigned matchEndDay = 0; + unsigned matchStartYear = 0; + unsigned matchStartMonth = 0; + unsigned matchStartDay = 0; + unsigned matchId = 0; + +public: + CSysInfoLoggerMsgFilter(const char *_source): matchSource(_source) + { + } + CSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *_source): matchSource(_source) + { + setMatchMsgId(msgId); + } + CSysInfoLoggerMsgFilter(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *_source) : + visibleOnly(_visibleOnly), hiddenOnly(_hiddenOnly), matchSource(_source) + { + if (hiddenOnly && visibleOnly) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: cannot filter by both hiddenOnly and visibleOnly"); + setDateRange(_year, _month, _day, _year, _month, _day); + } + virtual void setHiddenOnly() override + { + hiddenOnly = true; + } + virtual void setVisibleOnly() override + { + visibleOnly = true; + } + virtual void setMatchTimeStamp(timestamp_type ts) override + { + matchTimeStamp = ts; + } + virtual void setMatchSource(const char * source) override + { + matchSource.set(source); + } + virtual void setMatchCode(LogMsgCode code) override + { + matchCode = code; + } + virtual void setMatchAudience(LogMsgAudience audience) override + { + matchAudience = audience; + } + virtual void setMatchMsgClass(LogMsgClass msgClass) override + { + matchClass = msgClass; + } + virtual void setMatchMsgId(unsigned __int64 msgId) override + { + unsigned year, month, day, id; + decodeMessageId(msgId, year, month, day, id); + if (year==0 || month==0 || day==0 || id==0) + throw makeStringExceptionV(-1,"ISysInfoLoggerMsgFilter::setMatchMsgId invalid argument: %" I64F "u", msgId); + matchEndYear = matchStartYear = year; + matchEndMonth = matchStartMonth = month; + matchEndDay = matchStartDay = day; + matchId = id; + haveDateRange = false; + } + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) override + { + if ( (startDay && (!startMonth||!startYear)) || + (endDay && (!endMonth||!endYear)) ) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: month and year must be provided when filtering by day"); + if ((!startYear && startMonth) || (!endYear && endMonth)) + throw makeStringException(-1, "ISysInfoLoggerMsgFilter: year must be provided when filtering by month"); + // Make sure starts are on or before end dates + if ( (startYear > endYear) || (startMonth && (startYear == endYear && startMonth > endMonth)) + || (startDay && (startYear == endYear && startMonth == endMonth && startDay > endDay)) ) + throw makeStringExceptionV(-1, "ISysInfoLoggerMsgFilter: invalid date range: %04u-%02u-%02u to %04u-%02u-%02u", startYear, startMonth, startDay, endYear, endMonth, endDay); + matchEndYear = endYear; + matchEndMonth = endMonth; + matchEndDay = endDay; + matchStartYear = startYear; + matchStartMonth = startMonth; + matchStartDay = startDay; + if (matchEndYear||matchStartYear) + haveDateRange = (matchStartYearmatchEndYear) + return false; + if (tyear==matchEndYear) + { + if (matchEndMonth) + { + if (tmonth>matchEndMonth) + return false; + if (tmonth==matchEndMonth) + { + if (matchEndDay && tday>matchEndDay) + return false; + } + } + } + } + return true; + } + virtual bool queryHiddenOnly() const override + { + return hiddenOnly; + } + virtual bool queryVisibleOnly() const override + { + return visibleOnly; + } + virtual timestamp_type queryMatchTimeStamp() const override + { + return matchTimeStamp; + } + virtual unsigned queryStartYear() const override + { + return matchStartYear; + } + virtual unsigned queryStartMonth() const override + { + return matchStartMonth; + } + virtual unsigned queryStartDay() const override + { + return matchStartDay; + } + virtual unsigned queryEndYear() const override + { + return matchEndYear; + } + virtual unsigned queryEndMonth() const override + { + return matchEndMonth; + } + virtual unsigned queryEndDay() const override + { + return matchEndDay; + } + virtual const char * queryMatchSource() const override + { + return matchSource.str(); + } + virtual LogMsgCode queryMatchCode() const override + { + return matchCode; + } + virtual LogMsgAudience queryMatchAudience() const override + { + return matchAudience; + } + virtual LogMsgClass queryMatchClass() const override + { + return matchClass; + } + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const override + { + bool fullDayMatch=false; + bool hardMatchYear = matchStartYear && (matchStartYear==matchEndYear); + bool hardMatchMonth = matchStartMonth && (matchStartMonth==matchEndMonth); + if (hardMatchYear && hardMatchMonth) + { + // future: optimize when month unknown with "m%04u*" + xpath.appendf("m%04u%02u", matchStartYear, matchStartMonth); + if (matchStartDay==matchEndDay) + { + xpath.appendf("/d%02u", matchStartDay); + fullDayMatch = true; + } + } + if (fullDayMatch) + xpath.appendf("/" MSG_NODE); + else + xpath.appendf("//" MSG_NODE); + if (hiddenOnly) + xpath.append("[" ATTR_HIDDEN "='1')]"); + if (visibleOnly) + xpath.append("[" ATTR_HIDDEN "='0')]"); + if (!matchSource.isEmpty()) + xpath.appendf("[" ATTR_SOURCE "='%s']", matchSource.str()); + if (matchCode) + xpath.appendf("[" ATTR_CODE "='%d']", (int)matchCode); + if (matchAudience!=MSGAUD_all) + xpath.appendf("[" ATTR_AUDIENCE "='%s']", LogMsgAudienceToFixString(matchAudience)); + if (matchClass!=MSGCLS_all) + xpath.appendf("[" ATTR_CLASS "='%s']", LogMsgClassToFixString(matchClass)); + if (matchId) + xpath.appendf("[" ATTR_ID "='%u']", matchId); + if (matchTimeStamp) + xpath.appendf("[" ATTR_TIMESTAMP "='%" I64F "u']", matchTimeStamp); + return xpath; + } +}; + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source) +{ + return new CSysInfoLoggerMsgFilter(source); +} + +ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source) +{ + return new CSysInfoLoggerMsgFilter(msgId, source); +} + +class CSysInfoLoggerMsgIterator : public CSimpleInterfaceOf +{ + Linked filter; + // N.b. IRemoteConnection exists for the duration of the iterator so if this iterator exists for too long, it could cause + // performance issues for other clients: consider caching some messages and releasing connection (and reopening as necessary). + Owned conn; + bool updateable = false; + Owned msgIter; + CSysInfoLoggerMsg infoMsg; + + bool ensureMatch() + { + if (filter->hasDateRange()) + { + for (; msgIter->isValid(); msgIter->next()) + { + timestamp_type ts = msgIter->query().getPropInt64(ATTR_TIMESTAMP, 0); + if (filter->isInDateRange(ts)) + return true; + } + return false; + } + return msgIter->isValid(); + } + +public: + CSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * _filter, bool _updateable=false, IRemoteConnection *_conn=nullptr) : filter(_filter), updateable(_updateable), conn(_conn) + { + if (!conn) + { + unsigned mode = updateable ? RTM_LOCK_WRITE : RTM_LOCK_READ; + conn.setown(querySDS().connect(SYS_INFO_ROOT, myProcessSession(), mode, SDS_LOCK_TIMEOUT)); + if (!conn) + throw makeStringExceptionV(-1, "CSysInfoLoggerMsgIterator: unable to create connection to '%s'", SYS_INFO_ROOT); + } + } + CSysInfoLoggerMsg & queryInfoLoggerMsg() + { + return infoMsg.set(&(msgIter->get()), updateable); + } + virtual ISysInfoLoggerMsg & query() override + { + return queryInfoLoggerMsg(); + } + virtual bool first() override + { + StringBuffer xpath; + filter->getQualifierXPathFilter(xpath); + + msgIter.setown(conn->queryRoot()->getElements(xpath.str())); + if (!msgIter->first()) + return false; + return ensureMatch(); + } + virtual bool next() override + { + if (msgIter->next()) + return false; + return ensureMatch(); + } + virtual bool isValid() override + { + return msgIter ? msgIter->isValid() : false; + } +}; + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source) +{ + Owned filter = new CSysInfoLoggerMsgFilter(visibleOnly, hiddenOnly, year, month, day, source); + return new CSysInfoLoggerMsgIterator(filter, false); +} + +ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return new CSysInfoLoggerMsgIterator(msgFilter, false); +} + +// returns messageId +unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, timestamp_type ts) +{ + if (ts==0) + ts = getTimeStampNowValue(); + + if (isEmptyString(source)) + source = "unknown"; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_QUERY, SDS_LOCK_TIMEOUT); + if (!conn) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", SYS_INFO_ROOT); + + IPropertyTree * root = conn->queryRoot(); + unsigned id = root->getPropInt(ATTR_NEXTID, 1); + if (id==UINT_MAX) // wrap id to reuse id numbers (shouldn't wrap but no harm in doing this for safety) + id=1; + root->setPropInt(ATTR_NEXTID, id+1); + + StringBuffer xpath; + unsigned year, month, day; + extractDate(ts, year, month, day); + xpath.appendf("%s/m%04u%02u/d%02u/%s", SYS_INFO_ROOT, year, month, day, MSG_NODE); + Owned connMsgRoot = querySDS().connect(xpath.str(), myProcessSession(), RTM_CREATE_ADD, SDS_LOCK_TIMEOUT); + if (!connMsgRoot) + throw makeStringExceptionV(-1, "logSysInfoLogger: unable to create connection to '%s'", xpath.str()); + IPropertyTree * msgPT = connMsgRoot->queryRoot(); + + CSysInfoLoggerMsg sysInfoMsg(id, cat, code, source, msg, ts, false); + msgPT->setPropTree(nullptr, sysInfoMsg.getTree()); + msgPT->setProp(".", msg); // previous setPropTree doesn't set the node value + return makeMessageId(ts, id); +} + +unsigned updateMessage(IConstSysInfoLoggerMsgFilter * msgFilter, std::function updateOp) +{ + unsigned count = 0; + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, true); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + updateOp(sysInfoMsg); + ++count; + } + return count; +} + +unsigned updateMessage(unsigned __int64 msgId, const char *source, std::function updateOp) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, source); + return updateMessage(msgFilter, updateOp); +} + +unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);}); +} + +bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(true);})==1; +} + +unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + return updateMessage(msgFilter, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);}); +} + +bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + return updateMessage(msgId, source, [](CSysInfoLoggerMsg & sysInfoMsg){sysInfoMsg.setHidden(false);})==1; +} + +unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter) +{ + std::vector deleteXpathList; + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + { + Owned iter = new CSysInfoLoggerMsgIterator(msgFilter, false, conn.getLink()); + ForEach(*iter) + { + CSysInfoLoggerMsg & sysInfoMsg = iter->queryInfoLoggerMsg(); + StringBuffer xpath; + sysInfoMsg.getXpath(xpath); + deleteXpathList.push_back(xpath.str()); + } + } + IPropertyTree * root = conn->queryRoot(); + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())) + ++count; + } + return count; +} + +bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source) +{ + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, source); + return deleteLogSysInfoMsg(msgFilter); +} + +unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source) +{ + if (!year && month) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: year must be provided if month is specified (year=%u, month=%u, day=%u)", year, month, day); + if (!month && day) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: month must be provided if day is specified (year=%u, month=%u, day=%u)", year, month, day); + if (month>12) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid month(year=%u, month=%u, day=%u)", year, month, day); + if (day>31) + throw makeStringExceptionV(-1, "deleteOlderThanLogSysInfoMsg: invalid day(year=%u, month=%u, day=%u)", year, month, day); + // With visibleOnly/hiddenOnly option, use createSysInfoLoggerMsgFilter() + if (visibleOnly || hiddenOnly || day) + { + unsigned count = 0; + Owned msgFilter = createSysInfoLoggerMsgFilter(source); + if (hiddenOnly) + msgFilter->setHiddenOnly(); + if (visibleOnly) + msgFilter->setVisibleOnly(); + if (source) + msgFilter->setMatchSource(source); + msgFilter->setOlderThanDate(year, month, day); + return deleteLogSysInfoMsg(msgFilter); + } + + // With only date range, use this quicker method to remove whole subtrees + Owned conn = querySDS().connect(SYS_INFO_ROOT, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT); + if (!conn) + return 0; + + std::vector deleteXpathList; + IPropertyTree * root = conn->queryRoot(); + // future: optimize by getting only minimum set of subtrees to delete and get sorted elements(so search can stop earlier) + Owned monthIter = root->getElements("*"); + Owned innerException; //only first exception record/reported + ForEach(*monthIter) + { + IPropertyTree & monthPT = monthIter->query(); + if (year==0) + deleteXpathList.push_back(monthPT.queryName()); + else + { + unsigned msgYear = 0, msgMonth = 0; + const char *p = monthPT.queryName(); // should be in format 'myyyydd' + if (*p++ == 'm') + { + msgYear = readDigits(p, 4, false); + msgMonth = readDigits(p, 2, false); + } + if (msgYear == 0 || msgMonth == 0) + { + if (!innerException) + innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT " is invalid: %s", monthPT.queryName())); + continue; + } + if (msgYear > year) + continue; + if (msgYear < year) + deleteXpathList.push_back(monthPT.queryName()); + else + { + // msgYear matches year in this section + if (msgMonth > month) + continue; + else if (msgMonth < month) + deleteXpathList.push_back(monthPT.queryName()); + else // msgMonth==month + { + Owned dayIter = monthPT.getElements("*"); + ForEach(*dayIter) + { + IPropertyTree & dayPT = dayIter->query(); + unsigned msgDay = 0; + const char * d = dayPT.queryName(); + if (*d++ == 'd') + msgDay = readDigits(d, 2); + if (msgDay == 0) + { + if (!innerException) + innerException.setown(makeStringExceptionV(-1, "child of " SYS_INFO_ROOT "/%s is invalid: %s", monthPT.queryName(), dayPT.queryName())); + continue; + } + if (day && (msgDay >= day)) + continue; + + VStringBuffer xpath("%s/%s", monthPT.queryName(), dayPT.queryName()); + deleteXpathList.push_back(xpath.str()); + } + } + } + } + } + unsigned count = 0; + for (auto & xpath: deleteXpathList) + { + if (root->removeProp(xpath.c_str())); + ++count; + } + + if (innerException) // allow items to be deleted even if there is an exception + throw innerException.getClear(); + + return count; +} diff --git a/dali/base/sysinfologger.hpp b/dali/base/sysinfologger.hpp new file mode 100644 index 00000000000..2eb0af56edd --- /dev/null +++ b/dali/base/sysinfologger.hpp @@ -0,0 +1,92 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +#ifndef SYSINFOLOGGER +#define SYSINFOLOGGER + +#include "jlog.hpp" +#include "jutil.hpp" + +#ifdef DALI_EXPORT + #define SYSINFO_API DECL_EXPORT +#else + #define SYSINFO_API DECL_IMPORT +#endif + +interface ISysInfoLoggerMsg +{ + virtual bool queryIsHidden() const = 0; + virtual timestamp_type queryTimeStamp() const = 0; + virtual const char * querySource() const = 0; + virtual LogMsgCode queryLogMsgCode() const = 0; + virtual LogMsgAudience queryAudience() const = 0; + virtual LogMsgClass queryClass() const = 0; + virtual unsigned __int64 queryLogMsgId() const = 0; + virtual const char * queryMsg() const = 0; +}; + +interface IConstSysInfoLoggerMsgFilter : public IInterface +{ + virtual bool hasDateRange() const = 0; + virtual bool isInDateRange(timestamp_type ts) const = 0; + virtual bool queryHiddenOnly() const = 0; + virtual bool queryVisibleOnly() const = 0; + virtual timestamp_type queryMatchTimeStamp() const = 0; + virtual unsigned queryStartYear() const = 0; + virtual unsigned queryStartMonth() const = 0; + virtual unsigned queryStartDay() const = 0; + virtual unsigned queryEndYear() const = 0; + virtual unsigned queryEndMonth() const = 0; + virtual unsigned queryEndDay() const = 0; + virtual const char * queryMatchSource() const = 0; + virtual LogMsgCode queryMatchCode() const = 0; + virtual LogMsgAudience queryMatchAudience() const = 0; + virtual LogMsgClass queryMatchClass() const = 0; + virtual StringBuffer & getQualifierXPathFilter(StringBuffer & xpath) const = 0; +}; + +interface ISysInfoLoggerMsgFilter : extends IConstSysInfoLoggerMsgFilter +{ + virtual void setHiddenOnly() = 0; + virtual void setVisibleOnly() = 0; + virtual void setMatchTimeStamp(timestamp_type ts) = 0; + virtual void setMatchSource(const char * source) = 0; + virtual void setMatchCode(LogMsgCode code) = 0; + virtual void setMatchAudience(LogMsgAudience audience) = 0; + virtual void setMatchMsgClass(LogMsgClass msgClass) = 0; + virtual void setMatchMsgId(unsigned __int64 msgId) = 0; + virtual void setDateRange(unsigned startYear, unsigned startMonth, unsigned startDay, unsigned endYear, unsigned endMonth, unsigned endDay) = 0; + virtual void setOlderThanDate(unsigned year, unsigned month, unsigned day) = 0; +}; + +typedef IIteratorOf ISysInfoLoggerMsgIterator; + +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgFilter * createSysInfoLoggerMsgFilter(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(bool _visibleOnly, bool _hiddenOnly, unsigned _year, unsigned _month, unsigned _day, const char *source=nullptr); +SYSINFO_API ISysInfoLoggerMsgIterator * createSysInfoLoggerMsgIterator(ISysInfoLoggerMsgFilter * msgFilter); + +SYSINFO_API unsigned __int64 logSysInfoError(const LogMsgCategory & cat, LogMsgCode code, const char *source, const char * msg, unsigned __int64 ts); +SYSINFO_API unsigned hideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool hideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned unhideLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter, const char *source=nullptr); +SYSINFO_API bool unhideLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned deleteLogSysInfoMsg(IConstSysInfoLoggerMsgFilter * msgFilter); +SYSINFO_API bool deleteLogSysInfoMsg(unsigned __int64 msgId, const char *source=nullptr); +SYSINFO_API unsigned deleteOlderThanLogSysInfoMsg(bool visibleOnly, bool hiddenOnly, unsigned year, unsigned month, unsigned day, const char *source=nullptr); + +#endif diff --git a/system/jlib/jtime.cpp b/system/jlib/jtime.cpp index 87ed2384b7b..58dca7f0639 100644 --- a/system/jlib/jtime.cpp +++ b/system/jlib/jtime.cpp @@ -117,19 +117,6 @@ time_t timelocal(struct tm * local) #endif //__GNUC__ -static unsigned readDigits(char const * & str, unsigned numDigits) -{ - unsigned ret = 0; - while(numDigits--) - { - char c = *str++; - if(!isdigit(c)) - throwError1(JLIBERR_BadlyFormedDateTime, str); - ret = ret * 10 + (c - '0'); - } - return ret; -} - static void checkChar(char const * & str, char required) { char c = *str++; @@ -222,22 +209,32 @@ void CDateTime::set(time_t simple) void CDateTime::setString(char const * str, char const * * end, bool local) { + char const * beginstr = str; // save for error message if (!str||!*str) { clear(); return; } - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); - checkChar(str, 'T'); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); - unsigned nano = 0; + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0, nano = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + checkChar(str, 'T'); + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } + if(*str == '.') { unsigned digits; @@ -256,26 +253,44 @@ void CDateTime::setString(char const * str, char const * * end, bool local) void CDateTime::setDateString(char const * str, char const * * end) { - unsigned year = readDigits(str, 4); - checkChar(str, '-'); - unsigned month = readDigits(str, 2); - checkChar(str, '-'); - unsigned day = readDigits(str, 2); + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0; + try + { + year = readDigits(str, 4); + checkChar(str, '-'); + month = readDigits(str, 2); + checkChar(str, '-'); + day = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } if(end) *end = str; set(year, month, day, 0, 0, 0, 0, false); } void CDateTime::setTimeString(char const * str, char const * * end, bool local) { - unsigned year; - unsigned month; - unsigned day; + char const * beginstr = str; // save for error message + unsigned year = 0, month = 0, day = 0, hour = 0, minute = 0, sec = 0; getDate(year, month, day, false); - unsigned hour = readDigits(str, 2); - checkChar(str, ':'); - unsigned minute = readDigits(str, 2); - checkChar(str, ':'); - unsigned sec = readDigits(str, 2); + + try + { + hour = readDigits(str, 2); + checkChar(str, ':'); + minute = readDigits(str, 2); + checkChar(str, ':'); + sec = readDigits(str, 2); + } + catch (IException * e) + { + e->Release(); + throwError1(JLIBERR_BadlyFormedDateTime, beginstr); + } unsigned nano = 0; if(*str == '.') { @@ -668,10 +683,18 @@ void CScmDateTime::setString(const char * pstr) else if ((sign == '-') || (sign == '+')) { end++; - int delta = readDigits(end, 2); - if (*end++ != ':') + int delta = 0; + try + { + delta = readDigits(end, 2); + checkChar(end, ':'); + delta = delta * 60 + readDigits(end, 2); + } + catch (IException * e) + { + e->Release(); throwError1(JLIBERR_BadlyFormedDateTime, pstr); - delta = delta * 60 + readDigits(end, 2); + } if (sign == '-') delta = -delta; utcToLocalDelta = delta; diff --git a/system/jlib/jutil.cpp b/system/jlib/jutil.cpp index e2013d8399e..d22fda4888d 100644 --- a/system/jlib/jutil.cpp +++ b/system/jlib/jutil.cpp @@ -3632,3 +3632,20 @@ void hold(const char *msg) } + +unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure) +{ + unsigned ret = 0; + while (numDigits--) + { + char c = *str++; + if (!isdigit(c)) + { + if (throwOnFailure) + throw makeStringExceptionV(-1, "Invalid format (readDigits): %s", str); + return 0; + } + ret = ret * 10 + (c - '0'); + } + return ret; +} diff --git a/system/jlib/jutil.hpp b/system/jlib/jutil.hpp index 07c81dbc2cd..4841d61761b 100644 --- a/system/jlib/jutil.hpp +++ b/system/jlib/jutil.hpp @@ -655,6 +655,8 @@ struct HPCCBuildInfo extern jlib_decl HPCCBuildInfo hpccBuildInfo; extern jlib_decl bool checkCreateDaemon(unsigned argc, const char * * argv); +extern jlib_decl unsigned readDigits(char const * & str, unsigned numDigits, bool throwOnFailure = true); + //Createpassword of specified length, containing UpperCaseAlphas, LowercaseAlphas, numerics and symbols extern jlib_decl const char * generatePassword(StringBuffer &pwd, int pwdLen); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index 042546a3995..cbba197fbf4 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -37,6 +37,7 @@ #include #include "unittests.hpp" +#include "sysinfologger.hpp" //#define COMPAT @@ -3034,4 +3035,231 @@ class CFileNameNormalizeUnitTest : public CppUnit::TestFixture, CDfsLogicalFileN CPPUNIT_TEST_SUITE_REGISTRATION( CFileNameNormalizeUnitTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( CFileNameNormalizeUnitTest, "CFileNameNormalizeUnitTest" ); +#define SOURCE_COMPONENT_UNITTEST "sysinfologger-unittest" + +class DaliSysInfoLoggerTester : public CppUnit::TestFixture +{ + /* Note: global messages will be written for dates between 2000-02-04 and 2000-02-05 */ + /* Note: All global messages with time stamp before 2000-03-31 will be deleted */ + CPPUNIT_TEST_SUITE(DaliSysInfoLoggerTester); + CPPUNIT_TEST(testInit); + CPPUNIT_TEST(testSysInfoLogger); + CPPUNIT_TEST_SUITE_END(); + + struct TestCase + { + LogMsgCategory cat; + LogMsgCode code; + bool hidden; + const char * dateTimeStamp; + const char * msg; + }; + + std::vector testCases = + { + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-03T10:01:22.342343", + "CSysInfoLogger Unit test message 1" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-03T12:03:42.114233", + "CSysInfoLogger Unit test message 2" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + true, + "2000-02-03T14:02:13.678443", + "CSysInfoLogger Unit test message 3" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-03T16:05:18.8324832", + "CSysInfoLogger Unit test message 4" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42301, + false, + "2000-02-04T03:01:42.5754", + "CSysInfoLogger Unit test message 5" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42302, + false, + "2000-02-04T09:06:25.133132", + "CSysInfoLogger Unit test message 6" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42303, + false, + "2000-02-04T11:09:32.78439", + "CSysInfoLogger Unit test message 7" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T13:02:12.82821", + "CSysInfoLogger Unit test message 8" + }, + { + LogMsgCategory(MSGAUD_operator, MSGCLS_information, DefaultDetail), + 42304, + true, + "2000-02-04T18:32:11.23421", + "CSysInfoLogger Unit test message 9" + } + }; + + struct WrittenLogMessage + { + unsigned __int64 msgId; + unsigned __int64 ts; + unsigned testCaseIndex; + }; + std::vector writtenMessages; + + unsigned testRead(bool hiddenOnly=false, bool visibleOnly=false, unsigned year=0, unsigned month=0, unsigned day=0) + { + unsigned readCount=0; + try + { + std::set matchedMessages; // used to make sure every message written has been read back + Owned iter = createSysInfoLoggerMsgIterator(visibleOnly, hiddenOnly, year, month, day, SOURCE_COMPONENT_UNITTEST); + ForEach(*iter) + { + const ISysInfoLoggerMsg & sysInfoMsg = iter->query(); + + if (strcmp(sysInfoMsg.querySource(), SOURCE_COMPONENT_UNITTEST)!=0) + continue; // not a message written by this unittest so ignore + + // Check written message matches read message + unsigned __int64 msgId = sysInfoMsg.queryLogMsgId(); + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [msgId] (const auto & wm){ return (wm.msgId == msgId); }); + CPPUNIT_ASSERT_MESSAGE("Message read back not matching messages written by this unittest", matched!=writtenMessages.end()); + + // Make sure written messages matches message read back + matchedMessages.insert(matched->testCaseIndex); + TestCase & testCase = testCases[matched->testCaseIndex]; + ASSERT(testCase.hidden==sysInfoMsg.queryIsHidden()); + ASSERT(testCase.code==sysInfoMsg.queryLogMsgCode()); + ASSERT(strcmp(testCase.msg,sysInfoMsg.queryMsg())==0); + ASSERT(testCase.cat.queryAudience()==sysInfoMsg.queryAudience()); + ASSERT(testCase.cat.queryClass()==sysInfoMsg.queryClass()); + + readCount++; + } + ASSERT(readCount==matchedMessages.size()); // make sure there are no duplicates + } + catch (IException *e) + { + StringBuffer msg; + msg.appendf("testRead(hidden=%s, visible=%s) failed: ", boolToStr(hiddenOnly), boolToStr(visibleOnly)); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + return readCount; + } + +public: + ~DaliSysInfoLoggerTester() + { + daliClientEnd(); + } + void testInit() + { + daliClientInit(); + } + void testWrite() + { + writtenMessages.clear(); + unsigned testCaseIndex=0; + for (auto testCase: testCases) + { + try + { + CDateTime dateTime; + dateTime.setString(testCase.dateTimeStamp); + + unsigned __int64 ts = dateTime.getTimeStamp(); + unsigned __int64 msgId = logSysInfoError(testCase.cat, testCase.code, SOURCE_COMPONENT_UNITTEST, testCase.msg, ts); + writtenMessages.push_back({msgId, ts, testCaseIndex++}); + if (testCase.hidden) + { + Owned msgFilter = createSysInfoLoggerMsgFilter(msgId, SOURCE_COMPONENT_UNITTEST); + ASSERT(hideLogSysInfoMsg(msgFilter)==1); + } + } + catch (IException *e) + { + StringBuffer msg; + msg.append("logSysInfoError failed: "); + e->errorMessage(msg); + msg.appendf("(code %d)", e->errorCode()); + e->Release(); + CPPUNIT_FAIL(msg.str()); + } + } + ASSERT(testCases.size()==writtenMessages.size()); + } + void testSysInfoLogger() + { + // cleanup - remove messages that may have been left over from previous run + deleteOlderThanLogSysInfoMsg(false, false, 2001, 03, 00, SOURCE_COMPONENT_UNITTEST); + // Start of tests + testWrite(); + ASSERT(testRead(false, false)==9); + ASSERT(testRead(false, false, 2000, 02, 03)==4); + ASSERT(testRead(false, false, 2000, 02, 04)==5); + ASSERT(testRead(false, true)==5); //all visible messages + ASSERT(testRead(true, false)==4); //all hidden messages + ASSERT(deleteOlderThanLogSysInfoMsg(false, true, 2000, 02, 03, SOURCE_COMPONENT_UNITTEST)==2); + ASSERT(deleteOlderThanLogSysInfoMsg(true, false, 2000, 02, 04, SOURCE_COMPONENT_UNITTEST)==5); + + // testCase[7] and [8] are the only 2 remaining + // Delete single message test: delete testCase[7] + unsigned testCaseId = 7; + auto matched = std::find_if(writtenMessages.begin(), writtenMessages.end(), [testCaseId] (const auto & wm){ return (wm.testCaseIndex == testCaseId); }); + if (matched==writtenMessages.end()) + throw makeStringExceptionV(-1, "Can't find test case %u in written messages", testCaseId); + + Owned msgFilter = createSysInfoLoggerMsgFilter(matched->msgId, SOURCE_COMPONENT_UNITTEST); + ASSERT(deleteLogSysInfoMsg(msgFilter)==1); + + // Verify only 1 message remaining + ASSERT(testRead(false, false)==1); + // Delete 2000/02/04 and 2000/02/03 (one message but there are 2 parents remaining) + ASSERT(deleteOlderThanLogSysInfoMsg(false, false, 2000, 02, 05, SOURCE_COMPONENT_UNITTEST)==1); + // There shouldn't be any records remaining + ASSERT(testRead(false, false)==0); + + testWrite(); + + // delete all messages with MsgCode 42303 -> 3 messages + msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST)); + msgFilter->setMatchCode(42304); + ASSERT(deleteLogSysInfoMsg(msgFilter)==3); + + // delete all messages matching source=SOURCE_COMPONENT_UNITTEST + msgFilter.setown(createSysInfoLoggerMsgFilter(SOURCE_COMPONENT_UNITTEST)); + ASSERT(deleteLogSysInfoMsg(msgFilter)==6); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( DaliSysInfoLoggerTester ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( DaliSysInfoLoggerTester, "DaliSysInfoLoggerTester" ); + #endif // _USE_CPPUNIT diff --git a/testing/unittests/unittests.cpp b/testing/unittests/unittests.cpp index c11154dfe2c..dbe78e92f55 100644 --- a/testing/unittests/unittests.cpp +++ b/testing/unittests/unittests.cpp @@ -190,6 +190,7 @@ int main(int argc, const char *argv[]) excludeNames.append("*stress*"); excludeNames.append("*timing*"); excludeNames.append("*slow*"); + excludeNames.append("Dali*"); // disabled by default as dali not available when executed by smoketest } if (!includeNames.length()) From 5dfa8a192c328b5dc9749b47203742c8e39657b6 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Oct 2024 11:33:00 +0100 Subject: [PATCH 05/10] HPCC-32765 Report the compressed offset when reporting an error about an invalid compressed file Signed-off-by: Gavin Halliday --- system/jlib/jfcmp.hpp | 2 + system/jlib/jlz4.cpp | 12 +- system/jlib/jlzw.cpp | 65 ++++++----- system/jlib/jlzw.hpp | 2 +- testing/unittests/CMakeLists.txt | 1 + testing/unittests/filetests.cpp | 185 +++++++++++++++++++++++++++++++ 6 files changed, 238 insertions(+), 29 deletions(-) create mode 100644 testing/unittests/filetests.cpp diff --git a/system/jlib/jfcmp.hpp b/system/jlib/jfcmp.hpp index 24e6029df94..f9749a067f4 100644 --- a/system/jlib/jfcmp.hpp +++ b/system/jlib/jfcmp.hpp @@ -230,6 +230,7 @@ class jlib_decl CFcmpExpander : public CExpanderBase size32_t outlen; size32_t bufalloc; const size32_t *in = nullptr; + const void * original = nullptr; public: CFcmpExpander() @@ -247,6 +248,7 @@ class jlib_decl CFcmpExpander : public CExpanderBase virtual size32_t init(const void *blk) { + original = blk; const size32_t *expsz = (const size32_t *)blk; outlen = *expsz; in = (expsz+1); diff --git a/system/jlib/jlz4.cpp b/system/jlib/jlz4.cpp index 6a854c39fbf..930b38fa02d 100644 --- a/system/jlib/jlz4.cpp +++ b/system/jlib/jlz4.cpp @@ -264,6 +264,13 @@ class jlib_decl CLZ4Expander : public CFcmpExpander size32_t written; if (szchunk+totalExpanded outlen) - throwUnexpected(); + { + VStringBuffer msg("Decompression expected max %u bytes, but now %u at block offset %u", outlen, maxOut, (size32_t)((const byte *)in - (const byte *)original)); + throwUnexpectedX(msg.str()); + } maxOut += szchunk; // Likely to quickly approach the actual expanded size target.clear(); diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 6ff771eade6..6401657a7a9 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -2118,7 +2118,7 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface { unsigned code = e->errorCode(); StringBuffer msg; - e->errorMessage(msg).appendf(" at position %llu of %llu", nextExpansionPos, trailer.indexPos); + e->errorMessage(msg).appendf(" at uncompressed position %llu block %u of %llu", nextExpansionPos, curblocknum, trailer.indexPos); e->Release(); throw makeStringException(code, msg.str()); } @@ -2157,37 +2157,48 @@ class CCompressedFile : implements ICompressedFileIO, public CInterface void expand(const void *compbuf,MemoryBuffer &expbuf,size32_t expsize, offset_t compressedPos) { - size32_t rs = trailer.recordSize; - if (rs) { // diff expand - const byte *src = (const byte *)compbuf; - byte *dst = (byte *)expbuf.reserve(expsize); - if (expsize) { - assertex(expsize>=rs); - memcpy(dst,src,rs); - dst += rs; - src += rs; - expsize -= rs; - while (expsize) { + try + { + size32_t rs = trailer.recordSize; + if (rs) { // diff expand + const byte *src = (const byte *)compbuf; + byte *dst = (byte *)expbuf.reserve(expsize); + if (expsize) { assertex(expsize>=rs); - src += DiffExpand(src, dst, dst-rs, rs); - expsize -= rs; + memcpy(dst,src,rs); dst += rs; + src += rs; + expsize -= rs; + while (expsize) { + assertex(expsize>=rs); + src += DiffExpand(src, dst, dst-rs, rs); + expsize -= rs; + dst += rs; + } } } - } - else { // lzw or fastlz or lz4 - assertex(expander.get()); - size32_t exp = expander->expandFirst(expbuf, compbuf); - if (exp == 0) - { - unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf); - if (numZeros >= 16) - throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros); - } + else { // lzw or fastlz or lz4 + assertex(expander.get()); + size32_t exp = expander->expandFirst(expbuf, compbuf); + if (exp == 0) + { + unsigned numZeros = countZeros(trailer.blockSize, (const byte *)compbuf); + if (numZeros >= 16) + throw makeStringExceptionV(-1, "Unexpected zero fill in compressed file at position %llu length %u", compressedPos, numZeros); + } - startBlockPos = curblockpos; - nextExpansionPos = startBlockPos + exp; - fullBlockSize = expsize; + startBlockPos = curblockpos; + nextExpansionPos = startBlockPos + exp; + fullBlockSize = expsize; + } + } + catch (IException * e) + { + unsigned code = e->errorCode(); + StringBuffer msg; + e->errorMessage(msg).appendf(" at compressed position %llu of %llu", compressedPos, trailer.indexPos); + e->Release(); + throw makeStringException(code, msg.str()); } } diff --git a/system/jlib/jlzw.hpp b/system/jlib/jlzw.hpp index 1c3402d3318..da160d112e4 100644 --- a/system/jlib/jlzw.hpp +++ b/system/jlib/jlzw.hpp @@ -145,8 +145,8 @@ extern jlib_decl bool isCompressedFile(const char *filename); extern jlib_decl bool isCompressedFile(IFile *file); extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFile *file,IExpander *expander=NULL, bool memorymapped=false, IFEflags extraFlags=IFEnone); extern jlib_decl ICompressedFileIO *createCompressedFileReader(IFileIO *fileio,IExpander *expander=NULL); -extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4, IFEflags extraFlags=IFEnone); extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFileIO *fileio, bool append, size32_t recordsize,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4); +extern jlib_decl ICompressedFileIO *createCompressedFileWriter(IFile *file,size32_t recordsize,bool append=false,bool setcrc=true,ICompressor *compressor=NULL, unsigned compMethod=COMPRESS_METHOD_LZ4, IFEflags extraFlags=IFEnone); #define COMPRESSEDFILECRC (~0U) diff --git a/testing/unittests/CMakeLists.txt b/testing/unittests/CMakeLists.txt index 69a7b5e7a1c..e9ecc8ac297 100644 --- a/testing/unittests/CMakeLists.txt +++ b/testing/unittests/CMakeLists.txt @@ -45,6 +45,7 @@ set ( SRCS txSummarytests.cpp accessmaptests.cpp fxpptests.cpp + filetests.cpp ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp/fxpp/FragmentedXmlPullParser.cpp ${HPCC_SOURCE_DIR}/esp/bindings/SOAP/xpp/fxpp/FragmentedXmlAssistant.cpp ${CMAKE_BINARY_DIR}/generated/ws_loggingservice_esp.cpp diff --git a/testing/unittests/filetests.cpp b/testing/unittests/filetests.cpp new file mode 100644 index 00000000000..c2c29ea718e --- /dev/null +++ b/testing/unittests/filetests.cpp @@ -0,0 +1,185 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +/* + * File regression tests + * + */ + +#ifdef _USE_CPPUNIT +#include +#include +#include +#include "jsem.hpp" +#include "jfile.hpp" +#include "jdebug.hpp" +#include "jset.hpp" +#include "rmtfile.hpp" +#include "jlzw.hpp" +#include "jqueue.hpp" +#include "jregexp.hpp" +#include "jsecrets.hpp" +#include "jutil.hpp" +#include "junicode.hpp" + +#include "opentelemetry/sdk/common/attribute_utils.h" +#include "opentelemetry/sdk/resource/resource.h" + +#include "unittests.hpp" + +#define CPPUNIT_ASSERT_EQUAL_STR(x, y) CPPUNIT_ASSERT_EQUAL(std::string(x ? x : ""),std::string(y ? y : "")) + +class JlibFileTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(JlibFileTest); + CPPUNIT_TEST(testCompressed); + CPPUNIT_TEST(cleanup); + CPPUNIT_TEST_SUITE_END(); + + static constexpr const char * testFilename = "unittests_compressfile"; + void createCompressed() + { + Owned file(createIFile(testFilename)); + Owned io(createCompressedFileWriter(file, 0, false, false, nullptr)); + + constexpr size_t cnt = 10000; + constexpr size_t size = 1000; + offset_t pos = 0; + for (unsigned i = 0; i < cnt; i++) + { + byte temp[size]; + + for (unsigned j = 0; j < size; j += 4) + { + temp[j] = (byte)j; + temp[j+1] = (byte)j+1; + temp[j+2] = (byte)j+2; + temp[j+3] = (byte)random(); + } + + io->write(pos, size, temp); + pos += size; + } + } + void readCompressed(bool errorExpected) + { + bool success = false; + try + { + Owned file(createIFile(testFilename)); + Owned io(createCompressedFileReader(file)); + + constexpr size_t cnt = 10000; + constexpr size_t size = 1000; + offset_t pos = 0; + for (unsigned i = 0; i < cnt; i++) + { + byte temp[size]; + + io->read(pos, size, temp); + + for (unsigned j = 0; j < size; j += 4) + { + CPPUNIT_ASSERT_EQUAL(temp[j], (byte)j); + CPPUNIT_ASSERT_EQUAL(temp[j+1], (byte)(j+1)); + } + + pos += size; + } + + success = true; + } + catch (IException *e) + { + if (errorExpected) + { + DBGLOG(e, "Expected error reading compressed file:"); + } + else + { + StringBuffer msg("Unexpected error reading compressed file:"); + e->errorMessage(msg); + CPPUNIT_FAIL(msg.str()); + } + e->Release(); + } + if (success && errorExpected) + CPPUNIT_FAIL("Expected error reading compressed file"); + } + void read(offset_t offset, size32_t size, void * data) + { + Owned file(createIFile(testFilename)); + Owned io(file->open(IFOread)); + io->read(offset, size, data); + } + void write(offset_t offset, size32_t size, void * data) + { + Owned file(createIFile(testFilename)); + Owned io(file->open(IFOwrite)); + io->write(offset, size, data); + } + void testCompressed() + { + //patch the first block with zeros + constexpr byte zeros[0x100000] = { 0 }; + + createCompressed(); + readCompressed(false); + + write(0, sizeof(zeros), (void *)zeros); + readCompressed(true); + + createCompressed(); + write(0x10000, sizeof(zeros), (void *)zeros); + readCompressed(true); + + createCompressed(); + write(0x9000, sizeof(zeros), (void *)zeros); + readCompressed(true); + + //Test the second block being corrupted with zeros + size32_t firstBlockSize = 0; + createCompressed(); + read(4, sizeof(firstBlockSize), &firstBlockSize); + write(8+firstBlockSize, sizeof(zeros), (void *)zeros); + readCompressed(true); + + //Test the data after the second block being corrupted with zeros + createCompressed(); + read(4, sizeof(firstBlockSize), &firstBlockSize); + write(8+4+firstBlockSize, sizeof(zeros), (void *)zeros); + readCompressed(true); + + //Test the second block being corrupted to an invalid size + size32_t newSize = 1; + createCompressed(); + read(4, sizeof(firstBlockSize), &firstBlockSize); + write(8+firstBlockSize, sizeof(newSize), &newSize); + readCompressed(true); + } + void cleanup() + { + Owned file(createIFile(testFilename)); + file->remove(); + } +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( JlibFileTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JlibFileTest, "JlibFileTest" ); + +#endif From 226c107da5c565dde457b4dbeb3d767f888d5d93 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Oct 2024 11:31:58 +0100 Subject: [PATCH 06/10] HPCC-32764 Periodically log information about cpu-throttling Signed-off-by: Gavin Halliday --- system/jlib/jdebug.cpp | 112 ++++++++++++++++++++++++++++++-- system/jlib/jdebug.hpp | 7 ++ system/jlib/jstring.cpp | 34 ++++++++++ system/jlib/jstring.hpp | 18 ++++- testing/unittests/jlibtests.cpp | 17 +++++ 5 files changed, 181 insertions(+), 7 deletions(-) diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 4dcc7a141cf..5fa81ba1b96 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -816,6 +816,54 @@ static struct CNtKernelInformation } NtKernelFunctions; #endif +//=========================================================================== + +#ifndef _WIN32 + +static std::atomic gatheredGroup{false}; +static StringAttr cgroup; +static CriticalSection csgroupCs; +static const char * queryCGroup() +{ + if (!gatheredGroup) + { + CriticalBlock block(csgroupCs); + if (!gatheredGroup) + { + StringBuffer contents; + if (loadBinaryFile(contents, "/proc/self/cgroup", false)) + { + auto processLine = [](size_t len, const char * ln) + { + //Note ln points at the start of the line, but the line is not null terminated + switch (*ln) + { + case '0': + if (strncmp(ln, "0::/", 4) == 0) + { + //Format is 0::/ + //If not running in a container the "cgroup" may be something like user.slice/user-1000.slice/user@1000.service/.... + //If so ignore because it is not a real cgroup + if (!memchr(ln+4, '/', len-4)) + cgroup.set(ln+4, len-4); + } + break; + } + //Some systems with version 1 cgroups have :cpu,cpuacct:/ + const char * match = (const char *)jmemmem(len, ln, 14, ":cpu,cpuacct:/"); + if (match) + cgroup.set(match+14, (ln + len) - (match + 14)); + }; + + processLines(contents, processLine); + } + gatheredGroup = true; + } + } + return cgroup.get(); +} + +#endif //=========================================================================== @@ -893,6 +941,9 @@ SystemProcessInfo SystemProcessInfo::operator - (const SystemProcessInfo & rhs) result.activeDataMemory = activeDataMemory - rhs.activeDataMemory; result.majorFaults = majorFaults - rhs.majorFaults; result.numThreads = numThreads - rhs.numThreads; + result.numPeriods = numPeriods - rhs.numPeriods; + result.numThrottledPeriods = numThrottledPeriods - rhs.numThrottledPeriods; + result.timeThrottledNs = timeThrottledNs - rhs.timeThrottledNs; return result; } @@ -995,7 +1046,7 @@ bool ProcessInfo::update(unsigned flags) if (loadBinaryFile(contents, "/proc/self/status", false)) { contextSwitches = 0; - auto processLine = [this](const char * cur) + auto processLine = [this](size_t len, const char * cur) { __uint64 value; switch (*cur) @@ -1070,7 +1121,7 @@ bool SystemInfo::update(unsigned flags) StringBuffer contents; if (loadBinaryFile(contents, "/proc/stat", false)) { - auto processLine = [this](const char * ln) + auto processLine = [this](size_t len, const char * ln) { switch (*ln) { @@ -1105,6 +1156,52 @@ bool SystemInfo::update(unsigned flags) processLines(contents, processLine); } + auto processLine = [this](size_t len, const char * ln) + { + switch (*ln) + { + case 'n': + if (strncmp(ln, "nr_periods ", 11) == 0) + numPeriods = strtod(ln+11, nullptr); + else if (strncmp(ln, "nr_throttled ", 13) == 0) + numThrottledPeriods = strtod(ln+13, nullptr); + break; + case 't': + if (strncmp(ln, "throttled_usec ", 15) == 0) + timeThrottledNs = strtod(ln+15, nullptr) * 1000; + else if (strncmp(ln, "throttled_time ", 15) == 0) + timeThrottledNs = strtod(ln+15, nullptr); + break; + } + }; + + bool done = false; + const char * cgroup = queryCGroup(); + if (cgroup) + { + //Version 2 of cgroups has the information in cgroup/ + VStringBuffer filename("/sys/fs/cgroup/%s/cpu.stat", cgroup); + if (loadBinaryFile(contents.clear(), filename.str(), false)) + { + processLines(contents, processLine); + done = true; + } + else + { + //Some systems with version 1 cgroups have the information in /sys/fs/cgroup/cpu//cpu.stat + filename.clear().appendf("/sys/fs/cgroup/cpu/%s/cpu.stat", cgroup); + if (loadBinaryFile(contents.clear(), filename.str(), false)) + { + processLines(contents, processLine); + done = true; + } + } + } + + //If the version 2 file was not found look for ther version 1 information in cgroup/cpu + if (!done && loadBinaryFile(contents.clear(), "/sys/fs/cgroup/cpu/cpu.stat", false)) + processLines(contents, processLine); + return true; #endif } @@ -2502,9 +2599,16 @@ class CExtendedStats // Disk network and cpu stats } if (totalcpu) { - if (out.length()&&(out.charAt(out.length()-1)!=' ')) - out.append(' '); + ensureSeparator(out, ' '); out.appendf("CPU: usr=%d sys=%d iow=%d idle=%d", deltacpu.getUserPercent(), deltacpu.getSystemPercent(), deltacpu.getIoWaitPercent(), deltacpu.getIdlePercent()); + + __uint64 periods = deltacpu.getNumPeriods(); + if (periods) + { + unsigned throttling = deltacpu.getNumThrottledPeriods() * 100 / periods; + __uint64 timeThrottledNs = deltacpu.getTimeThrottledNs(); + out.appendf(" thr=%u%% thrns=%llu", throttling, timeThrottledNs); + } } return true; } diff --git a/system/jlib/jdebug.hpp b/system/jlib/jdebug.hpp index 0a8a0dacfcc..a4d5d41c705 100644 --- a/system/jlib/jdebug.hpp +++ b/system/jlib/jdebug.hpp @@ -446,6 +446,10 @@ class jlib_decl SystemProcessInfo unsigned getSystemPercent() const; unsigned getUserPercent() const; + __uint64 getNumPeriods() const { return numPeriods; } + __uint64 getNumThrottledPeriods() const { return numThrottledPeriods; } + __uint64 getTimeThrottledNs() const { return timeThrottledNs; } + __uint64 getTotal() const { return user + system + idle + iowait; } protected: __uint64 user = 0; // user time in jiffies (~`1/100s) @@ -461,6 +465,9 @@ class jlib_decl SystemProcessInfo __uint64 activeDataMemory = 0; __uint64 majorFaults = 0; __uint64 numThreads = 0; + __uint64 numPeriods = 0; + __uint64 numThrottledPeriods = 0; + __uint64 timeThrottledNs = 0; }; class jlib_decl ProcessInfo : public SystemProcessInfo diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp index 50951938ae5..7e581f2503a 100644 --- a/system/jlib/jstring.cpp +++ b/system/jlib/jstring.cpp @@ -2902,6 +2902,12 @@ void getSnakeCase(StringBuffer & out, const char * camelValue) } } +void ensureSeparator(StringBuffer & out, char separator) +{ + if (out.length() && (out.charAt(out.length()-1) != separator)) + out.append(separator); +} + /** * stristr - Case insensitive strstr() * @haystack: Where we will search for our @needle @@ -2949,3 +2955,31 @@ const char * stristr (const char *haystack, const char *needle) } return nullptr; } + + +const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle) +{ + if (lenNeedle == 0) + return haystack; + + if (lenHaystack < lenNeedle) + return nullptr; + + const char * search = (const char *)needle; + char first = *search; + if (lenNeedle == 1) + return memchr(haystack, first, lenHaystack); + + const char * buffer = (const char *)haystack; + for (size_t i = 0; i <= lenHaystack - lenNeedle; i++) + { + //Special case the first character to avoid a function call each iteration. + if (buffer[i] == first) + { + if (memcmp(buffer + i + 1, search + 1, lenNeedle-1) == 0) + return buffer + i; + } + } + + return nullptr; +} diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp index b3fe7651daf..d61b7fb7a1f 100644 --- a/system/jlib/jstring.hpp +++ b/system/jlib/jstring.hpp @@ -632,11 +632,18 @@ void processLines(const StringBuffer & content, LineProcessor process) const char * cur = content; while (*cur) { - process(cur); const char * next = strchr(cur, '\n'); - if (!next) + if (next) + { + if (next != cur) + process(next-cur, cur); + cur = next+1; + } + else + { + process(strlen(cur), cur); break; - cur = next+1; + } } } @@ -646,5 +653,10 @@ extern jlib_decl void processOptionString(const char * options, optionCallback c extern jlib_decl const char * stristr(const char *haystack, const char *needle); extern jlib_decl void getSnakeCase(StringBuffer & out, const char * camelValue); +//If the string has any characters, ensure the last character matches the separator +extern jlib_decl void ensureSeparator(StringBuffer & out, char separator); + +//Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own +extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle); #endif diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index ddf01c08461..398077ac3f4 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3390,6 +3390,7 @@ class MachineInfoTimingTest : public CppUnit::TestFixture DBGLOG(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)", (unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000), (unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches()); + DBGLOG(" Throttled: Periods(%llu/%llu) TimeNs(%llu)", deltaSystem.getNumThrottledPeriods(), deltaSystem.getNumPeriods(), deltaSystem.getTimeThrottledNs()); } for (unsigned j=0; j < i*100000000; j++) @@ -4630,6 +4631,7 @@ class JLibStringTest : public CppUnit::TestFixture public: CPPUNIT_TEST_SUITE(JLibStringTest); CPPUNIT_TEST(testStristr); + CPPUNIT_TEST(testMemMem); CPPUNIT_TEST_SUITE_END(); void testStristr() @@ -4647,6 +4649,21 @@ class JLibStringTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_STR(stristr("", "ABC"), ""); CPPUNIT_ASSERT_EQUAL_STR(stristr("ABC", ""), ""); } + + void testMemMem() + { + constexpr const char * haystack = "abcdefghijklmnopqrstuvwxyz"; + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 0, nullptr)); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(3, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(2, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+7), jmemmem(10, haystack, 3, "hij")); + CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(10, haystack, 3, "ijk")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 1, "i")); + CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(8, haystack, 1, "i")); + CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(9, haystack, 2, "ij")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 2, "ij")); + } }; CPPUNIT_TEST_SUITE_REGISTRATION( JLibStringTest ); From ce797e197142c5b61e5b583eedfd65d23785c83c Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:12:28 -0400 Subject: [PATCH 07/10] HPCC-32775 ECL Watch v9 fix Queries list status columns fix an issue where the status/icon columns of the Queries list were not properly displaying values of the results from the API call to ESP Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Queries.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/esp/src/src-react/components/Queries.tsx b/esp/src/src-react/components/Queries.tsx index ff45b2a7f6d..a64847aff70 100644 --- a/esp/src/src-react/components/Queries.tsx +++ b/esp/src/src-react/components/Queries.tsx @@ -120,8 +120,8 @@ export const Queries: React.FunctionComponent = ({ headerTooltip: nlsHPCC.ErrorWarnings, width: 16, sortable: false, - formatter: (error) => { - if (error > 0) { + formatter: (error, row) => { + if (row.ErrorCount > 0) { return ; } return ""; @@ -133,8 +133,9 @@ export const Queries: React.FunctionComponent = ({ headerTooltip: nlsHPCC.MixedNodeStates, width: 16, sortable: false, - formatter: (mixed) => { - if (mixed === true) { + formatter: (mixed, row) => { + const mixedStates = row.Clusters.ClusterQueryState[0]?.MixedNodeStates ?? false; + if (mixedStates === true) { return ; } return ""; @@ -144,8 +145,8 @@ export const Queries: React.FunctionComponent = ({ headerIcon: "SkypeCircleCheck", headerTooltip: nlsHPCC.Active, width: 16, - formatter: (activated) => { - if (activated === true) { + formatter: (activated, row) => { + if (row.Activated === true) { return ; } return ""; From f8e933af11714cdc63a1c8aa91a3754791b1ea8e Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 20 Sep 2024 18:24:55 -0400 Subject: [PATCH 08/10] HPCC-32532 ECL Watch v9 support for Grafana/Loki logging adds support to ECL Watch v9 for displaying logs from Grafana/Loki Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/package-lock.json | 8 ++-- esp/src/package.json | 2 +- esp/src/src-react/components/Logs.tsx | 60 ++++++++++++++++++--------- esp/src/src-react/hooks/platform.ts | 16 ++++++- esp/src/src/Utility.ts | 15 +++++++ 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 4fcce04e973..e54c6aba8ef 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -18,7 +18,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.96.1", + "@hpcc-js/comms": "2.97.0", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", @@ -2082,9 +2082,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.96.1", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.96.1.tgz", - "integrity": "sha512-38vIe8foZa5fYtrj65oeWyYWUDZmQTbKetHG5HXWZWMu0Lfmln8uG5/J7mO0ilw3ls2oZj7xOk5T/4xvg7v43w==", + "version": "2.97.0", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.97.0.tgz", + "integrity": "sha512-1AqZoYNDhb/zRLc/ZSj8eIrpOab1fUzMXFZYVYJvbDgO6dcwDn5cFMG+Y3g5m9Z/sc48E50S9lTZFFqQFZQOGg==", "dependencies": { "@hpcc-js/ddl-shim": "^2.21.0", "@hpcc-js/util": "^2.52.0", diff --git a/esp/src/package.json b/esp/src/package.json index db7708ef45d..2cb35d04463 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -44,7 +44,7 @@ "@hpcc-js/chart": "2.84.1", "@hpcc-js/codemirror": "2.63.0", "@hpcc-js/common": "2.72.0", - "@hpcc-js/comms": "2.96.1", + "@hpcc-js/comms": "2.97.0", "@hpcc-js/dataflow": "8.1.7", "@hpcc-js/eclwatch": "2.75.3", "@hpcc-js/graph": "2.86.0", diff --git a/esp/src/src-react/components/Logs.tsx b/esp/src/src-react/components/Logs.tsx index 0ec5a9b4699..a0ef17ee26b 100644 --- a/esp/src/src-react/components/Logs.tsx +++ b/esp/src/src-react/components/Logs.tsx @@ -3,7 +3,8 @@ import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluen import { GetLogsExRequest, LogaccessService, TargetAudience, LogType } from "@hpcc-js/comms"; import { Level, scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; -import { logColor, wuidToDate, wuidToTime } from "src/Utility"; +import { logColor, timestampToDate, wuidToDate, wuidToTime } from "src/Utility"; +import { useLoggingEngine } from "../hooks/platform"; import { HolyGrail } from "../layouts/HolyGrail"; import { pushParams } from "../util/history"; import { FluentGrid, useCopyButtons, useFluentStoreState, FluentColumns } from "./controls/Grid"; @@ -110,28 +111,49 @@ export const Logs: React.FunctionComponent = ({ const now = React.useMemo(() => new Date(), []); + const loggingEngine = useLoggingEngine(); + // Grid --- const columns = React.useMemo((): FluentColumns => { - return { - timestamp: { label: nlsHPCC.TimeStamp, width: 140, sortable: false, }, - message: { label: nlsHPCC.Message, width: 600, sortable: false, }, - components: { label: nlsHPCC.ContainerName, width: 150, sortable: false }, - instance: { label: nlsHPCC.PodName, width: 150, sortable: false }, - audience: { label: nlsHPCC.Audience, width: 60, sortable: false, }, - class: { - label: nlsHPCC.Class, width: 40, sortable: false, - formatter: level => { - const colors = logColor(levelMap(level)); - const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground }; - return {level}; - } + let retVal = { + timestamp: { + label: nlsHPCC.TimeStamp, width: 140, sortable: false, + formatter: ts => { + if (ts) { + if (ts.indexOf(":") < 0) { + return timestampToDate(ts).toISOString(); + } + return new Date(ts).toISOString(); + } + }, }, - workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, }, - processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, }, - logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, }, - threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, }, + message: { label: nlsHPCC.Message, width: 600, sortable: false, }, }; - }, [wuid]); + if (loggingEngine === "grafanacurl") { + retVal = Object.assign(retVal, { + pod: { label: nlsHPCC.PodName, width: 150, sortable: false }, + }); + } else { + retVal = Object.assign(retVal, { + instance: { label: nlsHPCC.PodName, width: 150, sortable: false }, + components: { label: nlsHPCC.ContainerName, width: 150, sortable: false }, + audience: { label: nlsHPCC.Audience, width: 60, sortable: false, }, + class: { + label: nlsHPCC.Class, width: 40, sortable: false, + formatter: level => { + const colors = logColor(levelMap(level)); + const styles = { backgroundColor: colors.background, padding: "2px 6px", color: colors.foreground }; + return {level}; + } + }, + workunits: { label: nlsHPCC.JobID, width: 50, sortable: false, hidden: wuid !== undefined, }, + processid: { label: nlsHPCC.ProcessID, width: 75, sortable: false, }, + logid: { label: nlsHPCC.Sequence, width: 70, sortable: false, }, + threadid: { label: nlsHPCC.ThreadID, width: 60, sortable: false, }, + }); + } + return retVal; + }, [loggingEngine, wuid]); const copyButtons = useCopyButtons(columns, selection, "logaccess"); diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index 66dac5d9487..4bc86caa2c4 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -2,7 +2,7 @@ import * as React from "react"; import { Octokit } from "octokit"; import { useConst } from "@fluentui/react-hooks"; import { scopedLogger } from "@hpcc-js/util"; -import { Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms"; +import { LogaccessService, Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms"; import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session"; import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo"; import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; @@ -10,6 +10,8 @@ import { Palette } from "@hpcc-js/common"; const logger = scopedLogger("src-react/hooks/platform.ts"); +export const service = new LogaccessService({ baseUrl: "" }); + declare const dojoConfig; export function useBuildInfo(): [BuildInfo, { isContainer: boolean, currencyCode: string, opsCategory: string }] { @@ -205,4 +207,14 @@ export function useModernMode(): { }, [modernMode, sessionStore, userStore]); return { modernMode, setModernMode }; -} \ No newline at end of file +} + +export function useLoggingEngine(): string { + const [loggingEngine, setLoggingEngine] = React.useState(""); + + React.useEffect(() => { + service.GetLogAccessInfo({}).then(response => setLoggingEngine(response.RemoteLogManagerType ?? "")); + }, []); + + return loggingEngine; +} \ No newline at end of file diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts index cc78350ed6d..4b70d873129 100644 --- a/esp/src/src/Utility.ts +++ b/esp/src/src/Utility.ts @@ -1242,6 +1242,21 @@ export function format(labelTpl, obj) { .join("\n") ; } + +const TEN_TRILLION = 10000000000000; +export function nanosToMillis(timestamp: number): number { + if (timestamp > TEN_TRILLION) { + return Math.round(timestamp / 1000000); + } else { + return timestamp; + } +} + +export function timestampToDate(timestamp: number): Date { + const millis = nanosToMillis(timestamp); + return new Date(millis); +} + const theme = getTheme(); const { semanticColors } = theme; From a1892863b840efcd9d5d493577ba209e8631bd92 Mon Sep 17 00:00:00 2001 From: Terrence Asselin Date: Tue, 24 Sep 2024 16:50:53 -0500 Subject: [PATCH 09/10] HPCC-30368 Add packagemap support for remoteStorage property Similar to how the daliip property is supported in the packagemap file, add support for a remoteStorage property that mirrors the `--remote-storage` CLI option. - Apply the same scoping rules as for daliip- values defined deeper in the hierarchy take precedence, and anything defined in the file takes precedence over a CLI option. - Disallow use of both daliip and remoteStorage in the same element. Signed-off-by: Terrence Asselin --- common/pkgfiles/referencedfilelist.cpp | 139 +++++++++++++++++-------- common/pkgfiles/referencedfilelist.hpp | 2 +- 2 files changed, 99 insertions(+), 42 deletions(-) diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp index 9a246966413..94e86efdc68 100644 --- a/common/pkgfiles/referencedfilelist.cpp +++ b/common/pkgfiles/referencedfilelist.cpp @@ -126,13 +126,43 @@ void splitDerivedDfsLocation(const char *address, StringBuffer &cluster, StringB prefix.append(basePrefix); } +void splitDerivedDfsLocationOrRemote(const char *address, StringBuffer &cluster, StringBuffer &ip, StringBuffer &prefix, + const char *defaultCluster, const char *baseCluster, const char *baseIP, const char *basePrefix, + const char *currentRemoteStorage, StringBuffer& effectiveRemoteStorage, const char *baseRemoteStorage) +{ + if (!isEmptyString(address) && !isEmptyString(currentRemoteStorage)) + throw makeStringExceptionV(-1, "Cannot specify both a dfs location (%s) and a remote storage location (%s)", address, currentRemoteStorage); + // Choose either a daliip (split from address) or a remote-storage location to + // propagate to the next level of parsing as the effective method of resolving a file + if (!isEmptyString(address)) + { + splitDfsLocation(address, cluster, ip, prefix, defaultCluster); + if (!ip.isEmpty()) + effectiveRemoteStorage.clear(); + return; + } + + if (!isEmptyString(currentRemoteStorage)) + { + effectiveRemoteStorage.append(currentRemoteStorage); + // Cluster and prefix aren't used if ip is empty, so they don't need to be cleared + ip.clear(); + return; + } + + ip.append(baseIP); + cluster.append(baseCluster); + prefix.append(basePrefix); + effectiveRemoteStorage.append(baseRemoteStorage); +} + class ReferencedFileList; class ReferencedFile : implements IReferencedFile, public CInterface { public: IMPLEMENT_IINTERFACE; - ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize) + ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize, const char *remoteStorageName) : pkgid(_pkgid), fileSize(0), numParts(0), flags(_flags), noDfsResolution(noDfs), calcFileSize(calcSize), trackSubFiles(false) { { @@ -142,6 +172,12 @@ class ReferencedFile : implements IReferencedFile, public CInterface } if (remoteStorage.length()) flags |= RefFileLFNRemote; + else if (!isEmptyString(remoteStorageName)) + { + // can be declared in packagemap at different scopes + remoteStorage.set(remoteStorageName); + flags |= RefFileLFNRemote; + } else if (daliip.length()) flags |= RefFileLFNForeign; else @@ -152,6 +188,9 @@ class ReferencedFile : implements IReferencedFile, public CInterface flags |= RefSubFile; } + ReferencedFile(const char *lfn, const char *sourceIP, const char *srcCluster, const char *prefix, bool isSubFile, unsigned _flags, const char *_pkgid, bool noDfs, bool calcSize) + : ReferencedFile(lfn, sourceIP, srcCluster, prefix, isSubFile, _flags, _pkgid, noDfs, calcSize, nullptr) { } + void reset() { flags &= ~(RefFileNotOnCluster | RefFileNotFound | RefFileResolvedForeign | RefFileResolvedRemote | RefFileCopyInfoFailed | RefFileCloned | RefFileNotOnSource); //these flags are calculated during resolve @@ -242,18 +281,18 @@ class ReferencedFileList : implements IReferencedFileList, public CInterface user.set(userDesc); } - void ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL); + void ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip=nullptr, const char *srcCluster=nullptr, const char *remotePrefix=nullptr, const char* remoteStorage=nullptr); - virtual void addFile(const char *ln, const char *daliip=NULL, const char *srcCluster=NULL, const char *remotePrefix=NULL); + virtual void addFile(const char *ln, const char *daliip=nullptr, const char *srcCluster=nullptr, const char *remotePrefix=nullptr, const char *remoteStorageName=nullptr); virtual void addFiles(StringArray &files); virtual void addFilesFromWorkUnit(IConstWorkUnit *cw); virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackageMap *pm, const char *queryid); virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg); virtual void addFilesFromPackageMap(IPropertyTree *pm); - void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix); - void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix); - void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster, const char *_remotePrefix); + void addFileFromSubFile(IPropertyTree &subFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName); + void addFilesFromSuperFile(IPropertyTree &superFile, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName); + void addFilesFromPackage(IPropertyTree &package, const char *_daliip, const char *srcCluster, const char *_remotePrefix, const char *_remoteStorageName); virtual IReferencedFileIterator *getFiles(); virtual void cloneFileInfo(StringBuffer &publisherWuid, const char *dstCluster, unsigned updateFlags, IDFUhelper *helper, bool cloneSuperInfo, bool cloneForeign, unsigned redundancy, unsigned channelsPerNode, int replicateOffset, const char *defReplicateFolder); @@ -883,12 +922,12 @@ class ReferencedFileIterator : implements IReferencedFileIterator, public CInter Owned iter; }; -void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip, const char *srcCluster, const char *prefix) +void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char *pkgid, bool noDfsResolution, const StringArray *subfileNames, const char *daliip, const char *srcCluster, const char *prefix, const char *remoteStorageName) { if (!allowForeign && checkForeign(ln)) throw MakeStringException(-1, "Foreign file not allowed%s: %s", (flags & RefFileInPackage) ? " (declared in package)" : "", ln); - Owned file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution, allowSizeCalc); + Owned file = new ReferencedFile(ln, daliip, srcCluster, prefix, false, flags, pkgid, noDfsResolution, allowSizeCalc, remoteStorageName); if (!file->logicalName.length()) return; if (subfileNames) @@ -904,9 +943,9 @@ void ReferencedFileList::ensureFile(const char *ln, unsigned flags, const char * } } -void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster, const char *prefix) +void ReferencedFileList::addFile(const char *ln, const char *daliip, const char *srcCluster, const char *prefix, const char *remoteStorageName) { - ensureFile(ln, 0, NULL, false, nullptr, daliip, srcCluster, prefix); + ensureFile(ln, 0, NULL, false, nullptr, daliip, srcCluster, prefix, remoteStorageName); } void ReferencedFileList::addFiles(StringArray &files) @@ -915,37 +954,41 @@ void ReferencedFileList::addFiles(StringArray &files) addFile(files.item(i)); } -void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *ip, const char *cluster, const char *prefix) +void ReferencedFileList::addFileFromSubFile(IPropertyTree &subFile, const char *ip, const char *cluster, const char *prefix, const char *remoteStorageName) { - addFile(subFile.queryProp("@value"), ip, cluster, prefix); + addFile(subFile.queryProp("@value"), ip, cluster, prefix, remoteStorageName); } -void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *_ip, const char *_cluster, const char *_prefix) +void ReferencedFileList::addFilesFromSuperFile(IPropertyTree &superFile, const char *_ip, const char *_cluster, const char *_prefix, const char *ancestorRemoteStorage) { StringBuffer ip; StringBuffer cluster; StringBuffer prefix; - splitDerivedDfsLocation(superFile.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix); + StringBuffer effectiveRemoteStorage; + splitDerivedDfsLocationOrRemote(superFile.queryProp("@daliip"), cluster, ip, prefix, nullptr, _cluster, _ip, _prefix, + superFile.queryProp("@remoteStorage"), effectiveRemoteStorage, ancestorRemoteStorage); if (superFile.hasProp("@sourceCluster")) cluster.set(superFile.queryProp("@sourceCluster")); Owned subFiles = superFile.getElements("SubFile[@value]"); ForEach(*subFiles) - addFileFromSubFile(subFiles->query(), ip, cluster, prefix); + addFileFromSubFile(subFiles->query(), ip, cluster, prefix, effectiveRemoteStorage); } -void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *_ip, const char *_cluster, const char *_prefix) +void ReferencedFileList::addFilesFromPackage(IPropertyTree &package, const char *_ip, const char *_cluster, const char *_prefix, const char *ancestorRemoteStorage) { StringBuffer ip; StringBuffer cluster; StringBuffer prefix; - splitDerivedDfsLocation(package.queryProp("@daliip"), cluster, ip, prefix, NULL, _cluster, _ip, _prefix); + StringBuffer effectiveRemoteStorage; + splitDerivedDfsLocationOrRemote(package.queryProp("@daliip"), cluster, ip, prefix, nullptr, _cluster, _ip, _prefix, + package.queryProp("@remoteStorage"), effectiveRemoteStorage, ancestorRemoteStorage); if (package.hasProp("@sourceCluster")) cluster.set(package.queryProp("@sourceCluster")); Owned supers = package.getElements("SuperFile"); ForEach(*supers) - addFilesFromSuperFile(supers->query(), ip, cluster, prefix); + addFilesFromSuperFile(supers->query(), ip, cluster, prefix, effectiveRemoteStorage); } void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm) @@ -953,20 +996,22 @@ void ReferencedFileList::addFilesFromPackageMap(IPropertyTree *pm) StringBuffer ip; StringBuffer cluster; StringBuffer prefix; + StringBuffer remoteStorageName; const char *srcCluster = pm->queryProp("@sourceCluster"); - splitDerivedDfsLocation(pm->queryProp("@daliip"), cluster, ip, prefix, srcCluster, srcCluster, NULL, NULL); + splitDerivedDfsLocationOrRemote(pm->queryProp("@daliip"), cluster, ip, prefix, srcCluster, srcCluster, nullptr, nullptr, + pm->queryProp("@remoteStorage"), remoteStorageName, nullptr); Owned packages = pm->getElements("Package"); ForEach(*packages) - addFilesFromPackage(packages->query(), ip, cluster, prefix); + addFilesFromPackage(packages->query(), ip, cluster, prefix, remoteStorageName); packages.setown(pm->getElements("Part/Package")); ForEach(*packages) - addFilesFromPackage(packages->query(), ip, cluster, prefix); + addFilesFromPackage(packages->query(), ip, cluster, prefix, remoteStorageName); } bool ReferencedFileList::addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg) { SummaryMap files; - if (cw->getSummary(SummaryType::ReadFile, files) && + if (cw->getSummary(SummaryType::ReadFile, files) && cw->getSummary(SummaryType::ReadIndex, files)) { for (const auto& [lName, summaryFlags] : files) @@ -1104,31 +1149,43 @@ void ReferencedFileList::resolveFiles(const StringArray &locations, const char * srcCluster.set(_srcCluster); remotePrefix.set(_remotePrefix); + ReferencedFileIterator files(this); + + // For use when expandSuperFiles=true if (useRemoteStorage) - { - DBGLOG("ReferencedFileList resolving remote storage files at %s", nullText(remoteLocation)); - if (!user) - user.setown(createUserDescriptor()); remoteStorage.set(remoteLocation); - ReferencedFileIterator files(this); - ForEach(files) - files.queryObject().resolveLocalOrRemote(locations, srcCluster, user, remoteStorage, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign); - } - else + + ForEach(files) { - if (!isEmptyString(remoteLocation)) - DBGLOG("ReferencedFileList resolving remote dali files at %s", remoteLocation); + ReferencedFile &file = files.queryObject(); + if (file.daliip.isEmpty() && (!file.remoteStorage.isEmpty() || useRemoteStorage)) + { + if (!user) + user.setown(createUserDescriptor()); + + if (file.remoteStorage.isEmpty()) // Can be set at multiple levels in a packagemap + file.remoteStorage.set(remoteLocation); // Top-level remoteLocation has lowest precedence, used if nothing set in packagemap + + DBGLOG("ReferencedFileList resolving remote storage file at %s", nullText(file.remoteStorage)); + file.resolveLocalOrRemote(locations, srcCluster, user, file.remoteStorage, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign); + } else - DBGLOG("ReferencedFileList resolving local files (no daliip)"); - remote.setown(!isEmptyString(remoteLocation) ? createINode(remoteLocation, 7070) : nullptr); + { + // The remoteLocation is a daliip when useRemoteStorage is false + const char *passedDaliip = !useRemoteStorage ? remoteLocation : nullptr; + if (!isEmptyString(passedDaliip) || !file.daliip.isEmpty()) + DBGLOG("ReferencedFileList resolving remote dali file at %s", isEmptyString(passedDaliip) ? nullText(file.daliip) : passedDaliip); + else + DBGLOG("ReferencedFileList resolving local file (no daliip or remote storage)"); + // Otherwise, passing nullptr for remote allows resolveLocalOrForeign to use ReferencedFile.daliip with + // the matching ReferencedFile.remotePrefix instead of the ReferencedFileList.remotePrefix passed in here. + remote.setown(!isEmptyString(passedDaliip) ? createINode(passedDaliip, 7070) : nullptr); + file.resolveLocalOrForeign(locations, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign); + } - ReferencedFileIterator files(this); - ForEach(files) - files.queryObject().resolveLocalOrForeign(locations, srcCluster, user, remote, remotePrefix, checkLocalFirst, expandSuperFiles ? &subfiles : NULL, trackSubFiles, resolveLFNForeign); + if (expandSuperFiles) + resolveSubFiles(subfiles, locations, checkLocalFirst, trackSubFiles, resolveLFNForeign); } - - if (expandSuperFiles) - resolveSubFiles(subfiles, locations, checkLocalFirst, trackSubFiles, resolveLFNForeign); } bool ReferencedFileList::filesNeedCopying(bool cloneForeign) diff --git a/common/pkgfiles/referencedfilelist.hpp b/common/pkgfiles/referencedfilelist.hpp index 3d20b40f7c3..41421b5a812 100644 --- a/common/pkgfiles/referencedfilelist.hpp +++ b/common/pkgfiles/referencedfilelist.hpp @@ -68,7 +68,7 @@ interface IReferencedFileList : extends IInterface virtual bool addFilesFromQuery(IConstWorkUnit *cw, const IHpccPackage *pkg)=0; virtual void addFilesFromPackageMap(IPropertyTree *pm)=0; - virtual void addFile(const char *ln, const char *daliip=NULL, const char *sourceProcessCluster=NULL, const char *remotePrefix=NULL)=0; + virtual void addFile(const char *ln, const char *daliip=nullptr, const char *sourceProcessCluster=nullptr, const char *remotePrefix=nullptr, const char *remoteStorageName=nullptr)=0; virtual void addFiles(StringArray &files)=0; virtual IReferencedFileIterator *getFiles()=0; From 85f74754f4e4d3fe5d3ca2e907cca8b401f4a493 Mon Sep 17 00:00:00 2001 From: M Kelly Date: Mon, 23 Sep 2024 15:58:35 -0400 Subject: [PATCH 10/10] HPCC-31755 Soapcall LOG multi-line separator Signed-off-by: M Kelly --- common/thorhelper/thorsoapcall.cpp | 63 +++++++++++++++++++++++++++--- common/thorhelper/thorsoapcall.hpp | 1 + ecl/hthor/hthor.cpp | 4 ++ roxie/ccd/ccdmain.cpp | 6 +++ thorlcr/graph/thgraph.cpp | 3 ++ thorlcr/thorutil/thormisc.hpp | 1 + 6 files changed, 72 insertions(+), 6 deletions(-) diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index f155bc069cf..c88b79db94a 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -50,6 +50,40 @@ using roxiemem::OwnedRoxieString; #define CONNECTION "Connection" unsigned soapTraceLevel = 1; +static StringBuffer soapSepString; + +void setSoapSepString(const char *_soapSepString) +{ + soapSepString.set(_soapSepString); +} + +static void multiLineAppendReplace(StringBuffer &origStr, StringBuffer &newStr) +{ + if (origStr.isEmpty()) + return; + + newStr.ensureCapacity(origStr.length()); + + const char *cursor = origStr; + while (*cursor) + { + switch (*cursor) + { + case '\r': + newStr.append(soapSepString); + if ('\n' == *(cursor+1)) + cursor++; + break; + case '\n': + newStr.append(soapSepString); + break; + default: + newStr.append(*cursor); + break; + } + ++cursor; + } +} #define WSCBUFFERSIZE 0x10000 #define MAXWSCTHREADS 50 //Max Web Service Call Threads @@ -1951,10 +1985,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { if (soapTraceLevel > 6 || master->logXML) { - if (!contentEncoded) - master->logctx.mCTXLOG("%s: request(%s)", master->wscCallTypeText(), request.str()); + StringBuffer contentStr; + if (contentEncoded) + contentStr.append(", content encoded."); + // Only do translation if soapcall LOG option set and soapSepString defined + if ( (master->logXML) && (soapSepString.length() > 0) ) + { + StringBuffer request2; + multiLineAppendReplace(request, request2); + master->logctx.CTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request2.str(), contentStr.str()); + } else - master->logctx.mCTXLOG("%s: request(%s), content encoded.", master->wscCallTypeText(), request.str()); + master->logctx.mCTXLOG("%s: request(%s)%s", master->wscCallTypeText(), request.str(), contentStr.str()); } } @@ -2250,9 +2292,18 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (checkContentDecoding(dbgheader, response, contentEncoding)) decodeContent(contentEncoding.str(), response); if (soapTraceLevel > 6 || master->logXML) - master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); - else if (soapTraceLevel > 8) - master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but... + { + // Only do translation if soapcall LOG option set and soapSepString defined + if ( (master->logXML) && (soapSepString.length() > 0) ) + { + StringBuffer response2; + multiLineAppendReplace(dbgheader, response2); + multiLineAppendReplace(response, response2); + master->logctx.CTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response2.str()); + } + else + master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); + } return rval; } diff --git a/common/thorhelper/thorsoapcall.hpp b/common/thorhelper/thorsoapcall.hpp index aa1c97a6c22..4ea42feef69 100644 --- a/common/thorhelper/thorsoapcall.hpp +++ b/common/thorhelper/thorsoapcall.hpp @@ -86,5 +86,6 @@ interface IRoxieAbortMonitor extern THORHELPER_API unsigned soapTraceLevel; extern THORHELPER_API IWSCHelper * createSoapCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor); extern THORHELPER_API IWSCHelper * createHttpCallHelper(IWSCRowProvider *, IEngineRowAllocator * outputAllocator, const char *authToken, SoapCallMode scMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor); +extern THORHELPER_API void setSoapSepString(const char *_soapSepString); #endif /* __THORSOAPCALL_HPP_ */ diff --git a/ecl/hthor/hthor.cpp b/ecl/hthor/hthor.cpp index e00fb39bcb7..610c15507cc 100644 --- a/ecl/hthor/hthor.cpp +++ b/ecl/hthor/hthor.cpp @@ -7738,6 +7738,10 @@ void CHThorWSCBaseActivity::init() JBASE64_Encode(uidpair.str(), uidpair.length(), authToken, false); } soapTraceLevel = agent.queryWorkUnit()->getDebugValueInt("soapTraceLevel", 1); + StringBuffer soapSepStr; + StringBufferAdaptor soapSepAdaptor(soapSepStr); + agent.queryWorkUnit()->getDebugValue("soapLogSepString", soapSepAdaptor); + setSoapSepString(soapSepStr.str()); } //--------------------------------------------------------------------------- diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index d86e3e0a410..bc17267cfb2 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -888,6 +888,12 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) udpTraceLevel = topology->getPropInt("@udpTraceLevel", runOnce ? 0 : 1); roxiemem::setMemTraceLevel(topology->getPropInt("@memTraceLevel", runOnce ? 0 : 1)); soapTraceLevel = topology->getPropInt("@soapTraceLevel", runOnce ? 0 : 1); + if (topology->hasProp("@soapLogSepString")) + { + StringBuffer tmpSepString; + topology->getProp("@soapLogSepString", tmpSepString); + setSoapSepString(tmpSepString.str()); + } miscDebugTraceLevel = topology->getPropInt("@miscDebugTraceLevel", 0); traceRemoteFiles = topology->getPropBool("@traceRemoteFiles", false); testAgentFailure = topology->getPropInt("expert/@testAgentFailure", testAgentFailure); diff --git a/thorlcr/graph/thgraph.cpp b/thorlcr/graph/thgraph.cpp index 414b2ec22e2..0b994cd1af3 100644 --- a/thorlcr/graph/thgraph.cpp +++ b/thorlcr/graph/thgraph.cpp @@ -2750,6 +2750,9 @@ void CJobBase::init() failOnLeaks = getOptBool(THOROPT_FAIL_ON_LEAKS); maxLfnBlockTimeMins = getOptInt(THOROPT_MAXLFN_BLOCKTIME_MINS, DEFAULT_MAXLFN_BLOCKTIME_MINS); soapTraceLevel = getOptInt(THOROPT_SOAP_TRACE_LEVEL, 1); + StringBuffer tmpSepString; + getOpt(THOROPT_SOAP_LOG_SEP_STRING, tmpSepString); + setSoapSepString(tmpSepString.str()); StringBuffer tracing("maxActivityCores = "); if (maxActivityCores) diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index 49d30f1e7aa..59d1797d1e8 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -118,6 +118,7 @@ #define THOROPT_MEMORY_SPILL_AT "memorySpillAt" // The threshold (%) that roxiemem will request memory to be reduced (default=80) #define THOROPT_FAIL_ON_LEAKS "failOnLeaks" // If any leaks are detected at the end of graph, fail the query (default=false) #define THOROPT_SOAP_TRACE_LEVEL "soapTraceLevel" // The trace SOAP level (default=1) +#define THOROPT_SOAP_LOG_SEP_STRING "soapLogSepString" // The SOAP request/response separator string for logging (default="") #define THOROPT_SORT_ALGORITHM "sortAlgorithm" // The algorithm used to sort records (quicksort/mergesort) #define THOROPT_COMPRESS_ALLFILES "v9_4_compressAllOutputs" // Compress all output files (default: bare-metal=off, cloud=on) #define THOROPT_AVOID_RENAME "avoidRename" // Avoid rename, write directly to target physical filenames (no temp file)