From 2e24ca18c55445fd17b495e51e6828b262d2c76a Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 14 Feb 2024 12:54:48 +0000 Subject: [PATCH 01/16] HPCC-31282 control:lockDali also "burns in" ephemeral roxie changes Signed-off-by: Richard Chapman --- roxie/ccd/ccd.hpp | 2 +- roxie/ccd/ccdmain.cpp | 10 ++++++---- roxie/ccd/ccdstate.cpp | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 1e793911e03..9fdcf3524ab 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -484,7 +484,7 @@ inline unsigned getBondedChannel(unsigned partNo) extern void FatalError(const char *format, ...) __attribute__((format(printf, 1, 2))); extern unsigned getNextInstanceId(); extern void closedown(); -extern void saveTopology(); +extern void saveTopology(bool lockDali); #define LOGGING_INTERCEPTED 0x01 #define LOGGING_TIMEACTIVITIES 0x02 diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 40f01b73e6b..e568c35a612 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -420,16 +420,18 @@ int myhook(int alloctype, void *, size_t nSize, int p1, long allocSeq, const uns } #endif -void saveTopology() +void saveTopology(bool lockDali) { - // Write back changes that have been made via certain control:xxx changes, so that they survive a roxie restart + // Write back changes that have been made via control:(un)lockDali changes, so that they survive a roxie restart // Note that they are overwritten when Roxie is manually stopped/started via hpcc-init service - these changes // are only intended to be temporary for the current session if (!useOldTopology) return; try { - saveXML(topologyFile.str(), topology); + Owned tempTopology = createPTreeFromXMLFile(topologyFile.str(), ipt_caseInsensitive); + tempTopology->setPropBool("@lockDali", lockDali); + saveXML(topologyFile.str(), tempTopology); } catch (IException *E) { @@ -696,7 +698,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) topologyFile.append(codeDirectory).append(PATHSEPCHAR).append("RoxieTopology.xml"); useOldTopology = checkFileExists(topologyFile.str()); topology = loadConfiguration(useOldTopology ? nullptr : defaultYaml, argv, "roxie", "ROXIE", topologyFile, nullptr, "@netAddress"); - saveTopology(); + saveTopology(topology->getPropBool("@lockDali", false)); // Any settings we read from topology that must NOT be overridden in workunit debug fields should be read at this point, before the following section getAllowedPipePrograms(allowedPipePrograms, true); diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp index 0387f243b83..c0aff8b68ac 100644 --- a/roxie/ccd/ccdstate.cpp +++ b/roxie/ccd/ccdstate.cpp @@ -2580,7 +2580,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme topology->setPropBool("@lockDali", true); if (daliHelper) daliHelper->disconnect(); - saveTopology(); + saveTopology(true); } else if (stricmp(queryName, "control:logfullqueries")==0) { @@ -3029,7 +3029,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme { topology->setPropBool("@lockDali", false); // Dali will reattach via the timer that checks every so often if can reattach... - saveTopology(); + saveTopology(false); } else if (stricmp(queryName, "control:unsuspend")==0) { From eb7ce56dc14922202095cf18411e28ac66ae8d29 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 14 Feb 2024 13:12:03 +0000 Subject: [PATCH 02/16] HPCC-31283 Use PING mechanism to detect mismatched configuration files Signed-off-by: Richard Chapman --- roxie/ccd/ccd.hpp | 4 ++++ roxie/ccd/ccdmain.cpp | 11 +++++++++++ roxie/ccd/ccdqueue.cpp | 14 ++++++++++++++ roxie/ccd/ccdstate.cpp | 16 +++++----------- 4 files changed, 34 insertions(+), 11 deletions(-) diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 9fdcf3524ab..d6b413aa715 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -485,6 +485,10 @@ extern void FatalError(const char *format, ...) __attribute__((format(printf, 1 extern unsigned getNextInstanceId(); extern void closedown(); extern void saveTopology(bool lockDali); +extern unsigned __int64 getTopologyHash(); + +extern unsigned __int64 currentTopologyHash; +extern unsigned __int64 originalTopologyHash; #define LOGGING_INTERCEPTED 0x01 #define LOGGING_TIMEACTIVITIES 0x02 diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index e568c35a612..a9ba2cc2d91 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -464,6 +464,16 @@ static std::vector> agentChannels; void *leakChecker = nullptr; // Used to deliberately leak an allocation to ensure leak checking is working +unsigned __int64 currentTopologyHash = 0; +unsigned __int64 originalTopologyHash = 0; + +hash64_t getTopologyHash() +{ + StringBuffer xml; + toXML(topology, xml, 0, XML_SortTags); + return rtlHash64Data(xml.length(), xml.str(), 707018); +} + #ifndef _CONTAINERIZED void readStaticTopology() { @@ -699,6 +709,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) useOldTopology = checkFileExists(topologyFile.str()); topology = loadConfiguration(useOldTopology ? nullptr : defaultYaml, argv, "roxie", "ROXIE", topologyFile, nullptr, "@netAddress"); saveTopology(topology->getPropBool("@lockDali", false)); + originalTopologyHash = currentTopologyHash = getTopologyHash(); // Any settings we read from topology that must NOT be overridden in workunit debug fields should be read at this point, before the following section getAllowedPipePrograms(allowedPipePrograms, true); diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index 401d2334969..36d30828f30 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -1041,6 +1041,8 @@ struct PingRecord { unsigned tick; IpAddress senderIP; + unsigned __int64 currentTopoHash; + unsigned __int64 originalTopoHash; }; void doPing(IRoxieQueryPacket *packet, const IRoxieContextLogger &logctx) @@ -1059,6 +1061,16 @@ void doPing(IRoxieQueryPacket *packet, const IRoxieContextLogger &logctx) StringBuffer s; throw MakeStringException(ROXIE_UNKNOWN_SERVER, "Message received from unknown Roxie server %s", header.toString(s).str()); } + if (originalTopologyHash != data->originalTopoHash) + { + StringBuffer s; + EXCLOG(MCoperatorError,"ERROR: Configuration file mismatch detected with Roxie server %s", header.toString(s).str()); + } + if (currentTopologyHash != data->currentTopoHash) + { + StringBuffer s; + DBGLOG("WARNING: Temporary configuration mismatch detected with Roxie server %s", header.toString(s).str()); + } RoxiePacketHeader newHeader(header, ROXIE_PING, 0); // subchannel not relevant Owned output = ROQ->createOutputStream(newHeader, true, logctx); void *ret = output->getBuffer(contextLength, false); @@ -3906,6 +3918,8 @@ class PingTimer : public Thread PingRecord data; data.senderIP.ipset(myNode.getIpAddress()); data.tick = usTick(); + data.originalTopoHash = originalTopologyHash; + data.currentTopoHash = currentTopologyHash; mb.append(sizeof(PingRecord), &data); if (doTrace(traceRoxiePings)) DBGLOG("PING sent"); diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp index c0aff8b68ac..f2b8ab240c2 100644 --- a/roxie/ccd/ccdstate.cpp +++ b/roxie/ccd/ccdstate.cpp @@ -2825,13 +2825,13 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme reply.appendf(""); else reply.appendf(""); - unsigned __int64 thash = getTopologyHash(); unsigned __int64 shash; { ReadLockBlock readBlock(packageCrit); shash = allQueryPackages->queryHash(); } - reply.appendf("", shash, thash); + reply.appendf("", + shash, currentTopologyHash, originalTopologyHash); } else if (stricmp(queryName, "control:resetcache")==0) { @@ -2936,13 +2936,13 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme reply.appendf(""); else reply.appendf(""); - unsigned __int64 thash = getTopologyHash(); unsigned __int64 shash; { ReadLockBlock readBlock(packageCrit); shash = allQueryPackages->queryHash(); } - reply.appendf("", shash, thash); + reply.appendf("", + shash, currentTopologyHash, originalTopologyHash); } else if (stricmp(queryName, "control:steppingEnabled")==0) { @@ -3068,6 +3068,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme unknown = true; break; } + currentTopologyHash = getTopologyHash(); if (unknown) throw MakeStringException(ROXIE_UNKNOWN_QUERY, "Unknown query %s", queryName); } @@ -3076,13 +3077,6 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme { throw MakeStringException(ROXIE_INVALID_INPUT, "Badly formated control query"); } - - hash64_t getTopologyHash() - { - StringBuffer xml; - toXML(topology, xml, 0, XML_SortTags); - return rtlHash64Data(xml.length(), xml.str(), 707018); - } }; extern IRoxieQueryPackageManagerSet *createRoxiePackageSetManager(const IQueryDll *standAloneDll) From 4fffc3ed80b36a52b3786495716dd708a79b009a Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 8 Mar 2024 13:47:53 +0000 Subject: [PATCH 03/16] HPCC-31421 Rationalize UNIMPLEMENTED and add a class variety Signed-off-by: Jake Smith --- fs/dafsserver/dafsserver.cpp | 13 +++++++++---- system/jlib/jexcept.cpp | 18 ++++++++++++++++++ system/jlib/jexcept.hpp | 15 +++++++++++---- system/jlib/jptree.cpp | 11 ++++++----- system/jlib/jtrace.cpp | 10 +++++----- 5 files changed, 49 insertions(+), 18 deletions(-) diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 5cb9a26b118..38f950fbaf3 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -1234,11 +1234,14 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf, } virtual void serializeCursor(MemoryBuffer &tgt) const override { - throwUnexpected(); + // we need to serialize something, because the lack of a cursor is used to signify end of stream + // NB: the cursor is opaque and only to be consumed by dafilesrv. When used it is simply passed back. + tgt.append("UNSUPPORTED"); } virtual void restoreCursor(MemoryBuffer &src) override { - throwUnexpected(); + throw makeStringExceptionV(0, "restoreCursor not supported in: %s", typeid(*this).name()); + throwUnimplemented(); } virtual void flushStatistics(CClientStats &stats) override { @@ -2389,11 +2392,13 @@ class CRemoteWriteBaseActivity : public CSimpleInterfaceOf } virtual void serializeCursor(MemoryBuffer &tgt) const override { - throwUnexpected(); + // we need to serialize something, because the lack of a cursor is used to signify end of stream + // NB: the cursor is opaque and only to be consumed by dafilesrv. When used it is simply passed back. + tgt.append("UNSUPPORTED"); } virtual void restoreCursor(MemoryBuffer &src) override { - throwUnexpected(); + throw makeStringExceptionV(0, "restoreCursor not supported in: %s", typeid(*this).name()); } virtual StringBuffer &getInfoStr(StringBuffer &out) const override { diff --git a/system/jlib/jexcept.cpp b/system/jlib/jexcept.cpp index d85aa4fbb82..1cbfb8bb150 100644 --- a/system/jlib/jexcept.cpp +++ b/system/jlib/jexcept.cpp @@ -799,6 +799,24 @@ void throwUnexpectedException(const char * what, const char * function, const ch throw makeStringExceptionV(9999, "Internal Error '%s' in %s() at %s(%d)", what, function, sanitizeSourceFile(file), line); } +void jlib_decl throwUnimplementedException(const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(9999, "UNIMPLEMENTED feature in function %s() at %s(%d)", function, sanitizeSourceFile(file), line); +} + +void jlib_decl throwUnimplementedException(const char * what, const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(-1, "UNIMPLEMENTED feature [%s] in function %s() at %s(%d)", what, function, sanitizeSourceFile(file), line); +} + +void jlib_decl throwUnimplementedException(const char * what, const char *what2, const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(-1, "UNIMPLEMENTED feature [%s %s] in function %s() at %s(%d)", what, what2, function, sanitizeSourceFile(file), line); +} + void raiseAssertException(const char *assertion, const char *file, unsigned line) { StringBuffer s; diff --git a/system/jlib/jexcept.hpp b/system/jlib/jexcept.hpp index 3e2dfb5b95b..3a80701c673 100644 --- a/system/jlib/jexcept.hpp +++ b/system/jlib/jexcept.hpp @@ -135,7 +135,7 @@ void jlib_decl setTerminateOnSEH(bool set=true); void jlib_decl setProcessAborted(bool _abortVal); __declspec(noreturn) void jlib_decl throwUnexpectedException(const char * function, const char * file, unsigned line) __attribute__((noreturn)); -__declspec(noreturn) void jlib_decl throwUnexpectedException(const char * function, const char * where, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnexpectedException(const char * what, const char * function, const char * file, unsigned line) __attribute__((noreturn)); const char jlib_decl *sanitizeSourceFile(const char *file); @@ -144,9 +144,16 @@ const char jlib_decl *sanitizeSourceFile(const char *file); #define throwUnexpectedX(x) throwUnexpectedException(x, __func__, sanitizeSourceFile(__FILE__), __LINE__) #define assertThrow(x) assertex(x) -#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define UNIMPLEMENTED_X(reason) throw makeStringExceptionV(-1, "UNIMPLEMENTED '" reason "' at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define UNIMPLEMENTED_XY(a,b) throw makeStringExceptionV(-1, "UNIMPLEMENTED " a " %s at %s(%d)", b, sanitizeSourceFile(__FILE__), __LINE__) +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * function, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * what, const char * function, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * what, const char *what2, const char * function, const char * file, unsigned line) __attribute__((noreturn)); +#define throwUnimplemented() throwUnimplementedException(__func__, sanitizeSourceFile(__FILE__), __LINE__) +#define throwUnimplementedX(x) throwUnimplementedException(x, __func__, sanitizeSourceFile(__FILE__), __LINE__) + +#define UNIMPLEMENTED throwUnimplementedException(__func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_C throwUnimplementedException("CLASSTYPE:", typeid(*this).name(), __func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_X(reason) throwUnimplementedException(reason, __func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_XY(a,b) throwUnimplementedException(a, b, __func__, sanitizeSourceFile(__FILE__), __LINE__) IException jlib_decl * deserializeException(MemoryBuffer & in); void jlib_decl serializeException(IException * e, MemoryBuffer & out); diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 994d916f422..3708a51d20d 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -48,8 +48,9 @@ #define WARNLEGACYCOMPARE #define XMLTAG_CONTENT "<>" -#undef UNIMPLEMENTED -#define UNIMPLEMENTED throw MakeIPTException(-1, "UNIMPLEMENTED") +#define UNIMPLEMENTED_IPT throw MakeIPTException(-1, "UNIMPLEMENTED feature in function %s() at %s(%d)", __func__, sanitizeSourceFile(__FILE__), __LINE__) + + #define CHECK_ATTRIBUTE(X) if (X && isAttribute(X)) throw MakeIPTException(PTreeExcpt_XPath_Unsupported, "Attribute usage invalid here"); #define AMBIGUOUS_PATH(X,P) { StringBuffer buf; buf.append(X": ambiguous xpath \"").append(P).append("\""); throw MakeIPTException(PTreeExcpt_XPath_Ambiguity,"%s",buf.str()); } @@ -1914,7 +1915,7 @@ bool PTree::renameProp(const char *xpath, const char *newName) if (strcmp(xpath,"/")==0) // rename of self allowed assuming no parent setName(newName); else if ('[' == *xpath) - UNIMPLEMENTED; + UNIMPLEMENTED_IPT; else if (isAttribute(xpath)) { StringBuffer val; @@ -3506,7 +3507,7 @@ bool PTree::checkPattern(const char *&xxpath) const for (;;) { if (matchElem->isBinary(tProp)) - UNIMPLEMENTED; + UNIMPLEMENTED_IPT; const char *rhs; unsigned rhslength; if (quoteEnd) @@ -6290,7 +6291,7 @@ class CStringBufferMarkupIOAdapter : public CInterfaceOf public: CStringBufferMarkupIOAdapter(StringBuffer &_out) : out(_out) { } virtual void flush() override { } - virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED; return 0; } + virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED_IPT; } virtual size32_t write(size32_t len, const void * data) override { out.append(len, (const char *)data); return len; } }; diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 1d619407d4f..feabb56304e 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -23,15 +23,10 @@ #include "opentelemetry/sdk/trace/batch_span_processor_factory.h" #include "opentelemetry/exporters/ostream/span_exporter_factory.h"// auto exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); #include "opentelemetry/exporters/ostream/common_utils.h" -//#define oldForEach ForEach // error: ‘ForEach’ was not declared in this scope -#undef ForEach //opentelemetry defines ForEach #include "opentelemetry/exporters/memory/in_memory_span_exporter_factory.h" #include "opentelemetry/trace/propagation/http_trace_context.h" //opentel_trace::propagation::kTraceParent -#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED #include "opentelemetry/trace/provider.h" //StartSpanOptions #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h" -#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define ForEach(i) for((i).first();(i).isValid();(i).next()) #include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h" #include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h" @@ -41,6 +36,11 @@ #include "opentelemetry/sdk/trace/exporter.h" #include "opentelemetry/sdk/trace/span_data.h" +// NB: undefine after opentelemetry includes, and before HPCC includes where we define. +#undef ForEach //opentelemetry defines ForEach +#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED + + #include "platform.h" #include "jlib.hpp" #include "jmisc.hpp" From c11fffcc42f87c0e929c1a22bb58d7c6732ab56c Mon Sep 17 00:00:00 2001 From: wangkx Date: Wed, 6 Mar 2024 09:41:47 -0500 Subject: [PATCH 04/16] HPCC-31194 Add dalilocks to WsDali In the dalidiag, the command '-locks' lists active SDS locks. The same function is added to the WsDali service. Signed-off-by: wangkx --- esp/scm/ws_dali.ecm | 7 ++++++- esp/services/ws_dali/ws_daliservice.cpp | 18 ++++++++++++++++++ esp/services/ws_dali/ws_daliservice.hpp | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/esp/scm/ws_dali.ecm b/esp/scm/ws_dali.ecm index 0404b14dadf..95f67e5eb97 100644 --- a/esp/scm/ws_dali.ecm +++ b/esp/scm/ws_dali.ecm @@ -156,6 +156,10 @@ ESPrequest [nil_remove] DisconnectClientConnectionRequest string Endpoint; }; +ESPrequest [nil_remove] ListSDSLocksRequest +{ +}; + ESPrequest [nil_remove] UnlockSDSLockRequest { string ConnectionID; @@ -180,7 +184,7 @@ ESPrequest [nil_remove] ClearTraceTransactionsRequest }; ESPservice [auth_feature("NONE"), //This declares that the method logic handles feature level authorization - version("1.06"), default_client_version("1.06"), exceptions_inline("./smc_xslt/exceptions.xslt")] WSDali + version("1.07"), default_client_version("1.07"), exceptions_inline("./smc_xslt/exceptions.xslt")] WSDali { ESPmethod [min_ver("1.01")] SetValue(SetValueRequest, ResultResponse); ESPmethod [min_ver("1.01")] GetValue(GetValueRequest, ResultResponse); @@ -205,6 +209,7 @@ ESPservice [auth_feature("NONE"), //This declares that the method logic handles ESPmethod [min_ver("1.05")] GetSDSStats(GetSDSStatsRequest, ResultResponse); ESPmethod [min_ver("1.05")] GetSDSSubscribers(GetSDSSubscribersRequest, ResultResponse); ESPmethod [min_ver("1.06")] DisconnectClientConnection(DisconnectClientConnectionRequest, ResultResponse); + ESPmethod [min_ver("1.07")] ListSDSLocks(ListSDSLocksRequest, ResultResponse); ESPmethod [min_ver("1.06")] UnlockSDSLock(UnlockSDSLockRequest, ResultResponse); ESPmethod [min_ver("1.06")] SaveSDSStore(SaveSDSStoreRequest, ResultResponse); ESPmethod [min_ver("1.06")] SetTraceTransactions(SetTraceTransactionsRequest, ResultResponse); diff --git a/esp/services/ws_dali/ws_daliservice.cpp b/esp/services/ws_dali/ws_daliservice.cpp index 9f602878120..5382963ea1f 100644 --- a/esp/services/ws_dali/ws_daliservice.cpp +++ b/esp/services/ws_dali/ws_daliservice.cpp @@ -619,6 +619,24 @@ bool CWSDaliEx::onDisconnectClientConnection(IEspContext& context, IEspDisconnec return true; } +bool CWSDaliEx::onListSDSLocks(IEspContext& context, IEspListSDSLocksRequest& req, IEspResultResponse& resp) +{ + try + { + checkAccess(context); + + Owned lockInfoCollection = querySDS().getLocks(); + StringBuffer result; + lockInfoCollection->toString(result); + resp.setResult(result); + } + catch(IException* e) + { + FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR); + } + return true; +} + bool CWSDaliEx::onUnlockSDSLock(IEspContext& context, IEspUnlockSDSLockRequest& req, IEspResultResponse& resp) { try diff --git a/esp/services/ws_dali/ws_daliservice.hpp b/esp/services/ws_dali/ws_daliservice.hpp index 3dc43d3d881..437cea434a1 100644 --- a/esp/services/ws_dali/ws_daliservice.hpp +++ b/esp/services/ws_dali/ws_daliservice.hpp @@ -73,6 +73,7 @@ class CWSDaliEx : public CWSDali virtual bool onGetSDSStats(IEspContext& context, IEspGetSDSStatsRequest& req, IEspResultResponse& resp) override; virtual bool onGetSDSSubscribers(IEspContext& context, IEspGetSDSSubscribersRequest& req, IEspResultResponse& resp) override; virtual bool onDisconnectClientConnection(IEspContext& context, IEspDisconnectClientConnectionRequest& req, IEspResultResponse& resp) override; + virtual bool onListSDSLocks(IEspContext& context, IEspListSDSLocksRequest& req, IEspResultResponse& resp) override; virtual bool onUnlockSDSLock(IEspContext& context, IEspUnlockSDSLockRequest& req, IEspResultResponse& resp) override; virtual bool onSaveSDSStore(IEspContext& context, IEspSaveSDSStoreRequest& req, IEspResultResponse& resp) override; virtual bool onSetTraceTransactions(IEspContext& context, IEspSetTraceTransactionsRequest& req, IEspResultResponse& resp) override; From 05455b8aa015c1ea86ddb1fb1621e6900052eff3 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Wed, 13 Mar 2024 12:28:47 +0000 Subject: [PATCH 05/16] HPCC-31448 xpath in CRemoteJsonReadActivity is set incorrectly --- fs/dafsserver/dafsserver.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 5cb9a26b118..a7009bda11a 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -2016,11 +2016,13 @@ class CRemoteJsonReadActivity : public CRemoteMarkupReadActivity public: CRemoteJsonReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc, TAKjsonread) { - xpath.set("/"); if (customRowTag.isEmpty()) // no override fileDesc->queryProperties().getProp("@rowTag", xpath); else + { + xpath.set("/"); xpath.append(customRowTag); + } } }; From 4c3aa47e23decea635337b484fb64645a3b1a259 Mon Sep 17 00:00:00 2001 From: Jack Del Vecchio Date: Wed, 6 Mar 2024 14:10:28 +0000 Subject: [PATCH 06/16] HPCC-30778 Migrate the xml reading code from dafilesrv to thorread.cpp --- common/thorhelper/thorread.cpp | 386 +++++++++++++++++++++++++++++++-- 1 file changed, 363 insertions(+), 23 deletions(-) diff --git a/common/thorhelper/thorread.cpp b/common/thorhelper/thorread.cpp index a5e7bce664e..5aebf0bc942 100644 --- a/common/thorhelper/thorread.cpp +++ b/common/thorhelper/thorread.cpp @@ -34,6 +34,7 @@ #include "thorcommon.hpp" #include "csvsplitter.hpp" #include "thormeta.hpp" +#include "thorxmlread.hpp" #ifdef _USE_PARQUET #include "parquetembed.hpp" @@ -814,6 +815,19 @@ class ExternalFormatDiskRowReader : public LocalDiskRowReader protected: virtual bool isBinary() const { return false; } + inline bool fieldFilterMatchProjected(const void * buffer) + { + if (projectedFilter.numFilterFields()) + { + unsigned numOffsets = projectedRecord->getNumVarFields() + 1; + size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t)); + RtlRow row(*projectedRecord, nullptr, numOffsets, variableOffsets); + row.setRow(buffer, 0); // Use lazy offset calculation + return projectedFilter.matches(row); + } + else + return true; + } protected: Owned fieldFetcher; @@ -880,22 +894,6 @@ class CsvDiskRowReader : public ExternalFormatDiskRowReader void processOption(CSVSplitter::MatchItem element, const IPropertyTree & csvOptions, const char * option, const char * dft, const char * dft2 = nullptr); - inline bool fieldFilterMatchProjected(const void * buffer) - { - if (projectedFilter.numFilterFields()) - { - unsigned numOffsets = projectedRecord->getNumVarFields() + 1; - size_t * variableOffsets = (size_t *)alloca(numOffsets * sizeof(size_t)); - RtlRow row(*projectedRecord, nullptr, numOffsets, variableOffsets); - row.setRow(buffer, 0); // Use lazy offset calculation - return projectedFilter.matches(row); - } - else - return true; - } - - size32_t getFixedDiskRecordSize(); - protected: constexpr static unsigned defaultMaxCsvRowSizeMB = 10; StringBuffer csvQuote, csvSeparate, csvTerminate, csvEscape; @@ -1054,24 +1052,364 @@ const void * CsvDiskRowReader::nextRow(MemoryBufferBuilder & builder) return nullptr; } +void CsvDiskRowReader::stop() +{ +} + +//--------------------------------------------------------------------------------------------------------------------- +class MarkupDiskRowReader : public ExternalFormatDiskRowReader, implements IXMLSelect +{ +private: + // JCSMORE - it would be good if these were cached/reused (anything using fetcher is single threaded) + class CFieldFetcher : public CSimpleInterfaceOf + { + unsigned numInputFields; + const RtlRecord &recInfo; + Linked currentMatch; + const char **compoundXPaths = nullptr; + const char *queryCompoundXPath(unsigned fieldNum) const + { + if (compoundXPaths && compoundXPaths[fieldNum]) + return compoundXPaths[fieldNum]; + else + return recInfo.queryXPath(fieldNum); + } + public: + CFieldFetcher(const RtlRecord &_recInfo, IColumnProvider *_currentMatch) : recInfo(_recInfo), currentMatch(_currentMatch) + { + numInputFields = recInfo.getNumFields(); -void CsvDiskRowReader::stop() + // JCSMORE - should this be done (optionally) when RtlRecord is created? + for (unsigned fieldNum=0; fieldNumqueryChildType()) + { + const char *xpath = recInfo.queryXPath(fieldNum); + dbgassertex(xpath); + const char *ptr = xpath; + char *expandedXPath = nullptr; + char *expandedXPathPtr = nullptr; + while (true) + { + if (*ptr == xpathCompoundSeparatorChar) + { + if (!compoundXPaths) + { + compoundXPaths = new const char *[numInputFields]; + memset(compoundXPaths, 0, sizeof(const char *)*numInputFields); + } + + size_t sz = strlen(xpath)+1; + expandedXPath = new char[sz]; + expandedXPathPtr = expandedXPath; + if (ptr == xpath) // if leading char, just skip + ++ptr; + else + { + size32_t len = ptr-xpath; + memcpy(expandedXPath, xpath, len); + expandedXPathPtr = expandedXPath + len; + *expandedXPathPtr++ = '/'; + ++ptr; + } + while (*ptr) + { + if (*ptr == xpathCompoundSeparatorChar) + { + *expandedXPathPtr++ = '/'; + ++ptr; + } + else + *expandedXPathPtr++ = *ptr++; + } + } + else + ptr++; + if ('\0' == *ptr) + { + if (expandedXPath) + { + *expandedXPathPtr = '\0'; + compoundXPaths[fieldNum] = expandedXPath; + } + break; + } + } + } + } + } + ~CFieldFetcher() + { + if (compoundXPaths) + { + for (unsigned fieldNum=0; fieldNumreadRaw(recInfo.queryXPath(fieldNum), rawSz); + sz = rawSz; + return (const byte *)ret; + } + virtual IDynamicRowIterator *getNestedIterator(unsigned fieldNum) const override + { + dbgassertex(fieldNum < numInputFields); + dbgassertex(currentMatch); + + const RtlRecord *nested = recInfo.queryNested(fieldNum); + if (!nested) + return nullptr; + + class CIterator : public CSimpleInterfaceOf + { + XmlChildIterator xmlIter; + Linked curFieldValueFetcher; + Linked parentMatch; + const RtlRecord &nestedRecInfo; + public: + CIterator(const RtlRecord &_nestedRecInfo, IColumnProvider *_parentMatch, const char *xpath) : nestedRecInfo(_nestedRecInfo), parentMatch(_parentMatch) + { + xmlIter.initOwn(parentMatch->getChildIterator(xpath)); + } + virtual bool first() override + { + IColumnProvider *child = xmlIter.first(); + if (!child) + { + curFieldValueFetcher.clear(); + return false; + } + curFieldValueFetcher.setown(new CFieldFetcher(nestedRecInfo, child)); + + return true; + } + virtual bool next() override + { + IColumnProvider *child = xmlIter.next(); + if (!child) + { + curFieldValueFetcher.clear(); + return false; + } + curFieldValueFetcher.setown(new CFieldFetcher(nestedRecInfo, child)); + return true; + } + virtual bool isValid() override + { + return nullptr != curFieldValueFetcher.get(); + } + virtual IDynamicFieldValueFetcher &query() override + { + assertex(curFieldValueFetcher); + return *curFieldValueFetcher; + } + }; + // JCSMORE - it would be good if these were cached/reused (can I assume anything using parent fetcher is single threaded?) + return new CIterator(*nested, currentMatch, queryCompoundXPath(fieldNum)); + } + virtual size_t getSize(unsigned fieldNum) const override { throwUnexpected(); } + virtual size32_t getRecordSize() const override { throwUnexpected(); } + }; + +public: + IMPLEMENT_IINTERFACE_USING(ExternalFormatDiskRowReader); + MarkupDiskRowReader(IDiskReadMapping * _mapping, ThorActivityKind _kind); + + virtual const void *nextRow() override; + virtual const void *nextRow(size32_t & resultSize) override; + virtual const void *nextRow(MemoryBufferBuilder & builder) override; + + virtual void stop() override; + virtual bool matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) override; + // IXMLSelect impl. + virtual void match(IColumnProvider &entry, offset_t startOffset, offset_t endOffset) { lastMatch.set(&entry); } + + bool checkOpen(); + IColumnProvider *queryMatch() const { return lastMatch; } + +protected: + virtual bool setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & expectedFilter) override; + +protected: + StringBuffer xpath; + StringBuffer rowTag; + + ThorActivityKind kind; + IXmlToRowTransformer *xmlTransformer; + Linked lastMatch; + Owned xmlParser; + + bool noRoot = false; + bool useXmlContents = false; + + const RtlRecord *record = nullptr; + bool opened = false; +}; + +MarkupDiskRowReader::MarkupDiskRowReader(IDiskReadMapping * _mapping, ThorActivityKind _kind) +: ExternalFormatDiskRowReader(_mapping), kind(_kind) { + const IPropertyTree & fileOptions = *mapping->queryFileOptions(); + const IPropertyTree & markupOptions = *fileOptions.queryPropTree("formatOptions"); + + markupOptions.getProp("ActivityOptions/rowTag", rowTag); + noRoot = markupOptions.getPropBool("noRoot"); + + record = &actualDiskMeta->queryRecordAccessor(true); } +bool MarkupDiskRowReader::setInputFile(IFile * inputFile, const char * _logicalFilename, unsigned _partNumber, offset_t _baseOffset, offset_t startOffset, offset_t length, const IPropertyTree * inputOptions, const FieldFilterArray & _expectedFilter) +{ + return ExternalFormatDiskRowReader::setInputFile(inputFile, _logicalFilename, _partNumber, _baseOffset, startOffset, length, inputOptions, _expectedFilter); +} -// IDiskRowReader +//Implementation of IAllocRowStream +const void *MarkupDiskRowReader::nextRow() +{ + checkOpen(); + while (xmlParser->next()) + { + if (lastMatch) + { + RtlDynamicRowBuilder builder(outputAllocator); + ((CFieldFetcher *)fieldFetcher.get())->setCurrentMatch(lastMatch); + size32_t sizeRead = translator->translate(builder, *this, *fieldFetcher); + dbgassertex(sizeRead); + lastMatch.clear(); + roxiemem::OwnedConstRoxieRow next = builder.finalizeRowClear(sizeRead); + + if (fieldFilterMatchProjected(next)) + return next.getClear(); + } + } + return eofRow; +} -size32_t CsvDiskRowReader::getFixedDiskRecordSize() +//Implementation of IRawRowStream +const void *MarkupDiskRowReader::nextRow(size32_t & resultSize) { - size32_t fixedDiskRecordSize = actualDiskMeta->getFixedSize(); - if (fixedDiskRecordSize && grouped) - fixedDiskRecordSize += 1; - return fixedDiskRecordSize; + tempOutputBuffer.clear(); + const void * next = nextRow(bufferBuilder); + resultSize = tempOutputBuffer.length(); + return next; } +const void * MarkupDiskRowReader::nextRow(MemoryBufferBuilder & builder) +{ + checkOpen(); + while (xmlParser->next()) + { + if (lastMatch) + { + ((CFieldFetcher *)fieldFetcher.get())->setCurrentMatch(lastMatch); + size32_t resultSize = translator->translate(builder, *this, *fieldFetcher); + dbgassertex(resultSize); + lastMatch.clear(); + const void *ret = builder.getSelf(); + + if (fieldFilterMatchProjected(ret)) + { + builder.finishRow(resultSize); + return ret; + } + else + builder.removeBytes(resultSize); + } + } + return eofRow; +} + +void MarkupDiskRowReader::stop() +{ + opened = false; +} + +bool MarkupDiskRowReader::matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) +{ + return ExternalFormatDiskRowReader::matches(format, streamRemote, otherMapping); +} + +bool MarkupDiskRowReader::checkOpen() +{ + class CSimpleStream : public CSimpleInterfaceOf + { + Linked stream; + public: + CSimpleStream(ISerialStream *_stream) : stream(_stream) + { + } + // ISimpleReadStream impl. + virtual size32_t read(size32_t max_len, void * data) override + { + size32_t got; + const void *res = stream->peek(max_len, got); + if (got) + { + if (got>max_len) + got = max_len; + memcpy(data, res, got); + stream->skip(got); + } + return got; + } + }; + + if (!opened) + { + Owned simpleStream = new CSimpleStream(inputStream); + if (kind==TAKjsonread) + xmlParser.setown(createJSONParse(*simpleStream, xpath, *this, noRoot?ptr_noRoot:ptr_none, useXmlContents)); + else + xmlParser.setown(createXMLParse(*simpleStream, xpath, *this, noRoot?ptr_noRoot:ptr_none, useXmlContents)); + + if (!fieldFetcher) + fieldFetcher.setown(new CFieldFetcher(*record, nullptr)); + + opened = true; + return true; + } + return false; +} + + +class XmlDiskRowReader : public MarkupDiskRowReader +{ +public: + XmlDiskRowReader(IDiskReadMapping * _mapping) + : MarkupDiskRowReader(_mapping, TAKxmlread) + { + const IPropertyTree & fileOptions = *mapping->queryFileOptions(); + const IPropertyTree & xmlOptions = *fileOptions.queryPropTree("formatOptions"); + + if (rowTag.isEmpty()) // no override + xmlOptions.getProp("xpath", xpath); + else + { + xpath.set("/Dataset/"); + xpath.append(rowTag); + } + } + + bool matches(const char * format, bool streamRemote, IDiskReadMapping * otherMapping) + { + if (!strieq(format, "xml")) + return false; + return MarkupDiskRowReader::matches(format, streamRemote, otherMapping); + } +}; //--------------------------------------------------------------------------------------------------------------------- @@ -1381,6 +1719,7 @@ const void * ParquetDiskRowReader::nextRow() // Row is built in temporary buffer and reused. const void * ParquetDiskRowReader::nextRow(size32_t & resultSize) { + tempOutputBuffer.clear(); const void * next = nextRow(bufferBuilder); resultSize = tempOutputBuffer.length(); return next; @@ -1745,6 +2084,7 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) // that creates the appropriate disk row reader object genericFileTypeMap.emplace("flat", [](IDiskReadMapping * _mapping) { return new BinaryDiskRowReader(_mapping); }); genericFileTypeMap.emplace("csv", [](IDiskReadMapping * _mapping) { return new CsvDiskRowReader(_mapping); }); + genericFileTypeMap.emplace("xml", [](IDiskReadMapping * _mapping) { return new XmlDiskRowReader(_mapping); }); #ifdef _USE_PARQUET genericFileTypeMap.emplace(PARQUET_FILE_TYPE_NAME, [](IDiskReadMapping * _mapping) { return new ParquetDiskRowReader(_mapping); }); #endif From a39d8995b109272ff569ec3f9701d6e66ef3f6e4 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 14 Mar 2024 17:07:57 +0000 Subject: [PATCH 07/16] Split off 9.2.70 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 639d553b742..aa1b38189bb 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.69-closedown0 +version: 9.2.71-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.69-closedown0 +appVersion: 9.2.71-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index b57b5a84b92..86da4d22917 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 29e76b501cd..b0d572e455c 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index e2722829484..4a75d2a2783 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 6e608619a5a..998fc1a3fe0 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index b26989c9cdc..dd9b93f6a9a 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index d20bb478117..599f1f6f3cc 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 69ae68f7331..9034e68b3b5 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 3fc8e209313..bf65b2ee2d5 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 459825ad1e8..d938b423316 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 71a30393f2f..4c1c0d6eed8 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 6660e282188..7360bca9bc1 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 0d95e79a84f..f43ef3cb10e 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.69-closedown0 + helmVersion: 9.2.71-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 5323e18ba8f..db88b740382 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 69 ) +set ( HPCC_POINT 71 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-03-07T17:27:44Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-03-14T17:07:57Z" ) ### From 1c441cae730ff7249d67905873d1f9769f08fc0d Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 14 Mar 2024 17:09:19 +0000 Subject: [PATCH 08/16] Split off 9.4.44 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 8ddd130af63..c5acfc861ac 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.43-closedown0 +version: 9.4.45-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.43-closedown0 +appVersion: 9.4.45-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 4aea93c6980..9ffdad96f05 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1502,7 +1502,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 8314f27b730..f33c9551083 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 8f940717307..c039cd4171b 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -84,7 +84,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 55ab4834628..490e1902723 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 3ca1121f043..0b4ecd40586 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 68000adb1bc..03990cc7ed0 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 6a0d68385dc..daf883ef859 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index a98f626918f..e57db210b3a 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -122,7 +122,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 42534212775..3662026dfad 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 6ac27dc43c9..c663c4717c0 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -183,7 +183,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -245,7 +245,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -353,7 +353,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index bbb75f14322..3405a4eec2f 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 6dba5ea1e6d..fad977de83a 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.4.43-closedown0 + helmVersion: 9.4.45-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 1952e234114..d03bdae449b 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 43 ) +set ( HPCC_POINT 45 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-03-07T17:32:39Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-03-14T17:09:18Z" ) ### From c442d91bc27a6af45de6f84f2313c1a7d059934e Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 15 Mar 2024 09:14:29 -0400 Subject: [PATCH 09/16] HPCC-31447 ECL Watch v9 fix dropdown widths correct the default behaviour for FluentUI v8 dropdowns to truncate option text to the width of the input, instead fitting to the content Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/forms/Fields.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index a25c79a2c77..d9221b96775 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -136,8 +136,8 @@ const AsyncDropdown: React.FunctionComponent = ({ }, [onChange, selectedItem, selectedIdx, selectedKey]); return options === undefined ? - : - setSelectedItem(item)} placeholder={placeholder} disabled={disabled} required={required} errorMessage={errorMessage} className={className} />; + : + setSelectedItem(item)} placeholder={placeholder} disabled={disabled} required={required} errorMessage={errorMessage} className={className} />; }; interface DropdownMultiProps { From ad83e18b26e7f117c6a87e964d5b35db7812fb99 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 15 Mar 2024 10:46:14 -0400 Subject: [PATCH 10/16] HPCC-31354 ECL Watch v9 display ECL for failed WUs For failed WUs, /WsWorkunits/WUFile wouldn't return a usable result, so on the ECL tab of the WU Details page fetch the ECL text via WUInfo Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/ECLArchive.tsx | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 92e230cfd67..ca4ee1de6ae 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -1,6 +1,7 @@ import * as React from "react"; import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react"; -import { WUDetails, IScope } from "@hpcc-js/comms"; +import { Workunit, WUDetails, IScope } from "@hpcc-js/comms"; +import { scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; import { useWorkunitArchive } from "../hooks/workunit"; import { useWorkunitMetrics } from "../hooks/metrics"; @@ -12,6 +13,8 @@ import { ECLArchiveTree } from "./ECLArchiveTree"; import { ECLArchiveEditor } from "./ECLArchiveEditor"; import { MetricsPropertiesTables } from "./MetricsPropertiesTables"; +const logger = scopedLogger("src-react/components/ECLArchive.tsx"); + const scopeFilterDefault: WUDetails.RequestNS.ScopeFilter = { MaxDepth: 999999, ScopeTypes: ["graph"] @@ -54,8 +57,15 @@ export const ECLArchive: React.FunctionComponent = ({ setSelectionText(archive?.content(selection) ?? ""); setMarkers(archive?.markers(selection) ?? []); setSelectedMetrics(archive?.metrics(selection) ?? []); + } else { + if (archive && !archive.build) { + const wu = Workunit.attach({ baseUrl: "" }, wuid); + wu.fetchQuery().then(function (query) { + setSelectionText(query?.Text ?? ""); + }).catch(err => logger.error(err)); + } } - }, [archive, metrics.length, selection]); + }, [archive, metrics.length, selection, wuid]); const setSelectedItem = React.useCallback((selId: string) => { pushUrl(`${parentUrl}/${selId}`); From 9e61db7f1b01d36bcf0c4e7b7d47018bb29b50ea Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 18 Mar 2024 16:49:21 +0000 Subject: [PATCH 11/16] HPCC-31484 Remove spurious newlines from thor tracing Signed-off-by: Gavin Halliday --- thorlcr/master/thgraphmanager.cpp | 2 +- thorlcr/slave/slavmain.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 4cf6300e651..a8384c6d237 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1092,7 +1092,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, SCMStringBuffer eclstr; StringAttr user(workunit.queryUser()); - PROGLOG("Started wuid=%s, user=%s, graph=%s\n", wuid.str(), user.str(), graphName); + PROGLOG("Started wuid=%s, user=%s, graph=%s", wuid.str(), user.str(), graphName); PROGLOG("Query %s loaded", soPath.str()); Owned job = createThorGraph(graphName, workunit, querySo, sendSo, agentEp); diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index c9acc3303a5..80f730d83cf 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1897,7 +1897,7 @@ class CJobListener : public CSimpleInterface thorJob.setJobID(thorJobId); setDefaultJobId(thorJobId); - PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]\n", wuid.get(), user.str(), graphName.get(), maxLogDetail); + PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]", wuid.get(), user.str(), graphName.get(), maxLogDetail); PROGLOG("Using query: %s", soPath.str()); if (!getExpertOptBool("slaveDaliClient") && workUnitInfo->getPropBool("Debug/slavedaliclient", false)) From de8d738eebccc4ce1be0a76bb9483b4533cd5fb6 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 19 Mar 2024 12:29:31 +0000 Subject: [PATCH 12/16] HPCC-31486 Prevent newly resolved secrets from being updated too early Signed-off-by: Gavin Halliday --- system/jlib/jsecrets.cpp | 18 ++++++++++-------- testing/unittests/jlibtests.cpp | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index f731e97c987..4573196469a 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -425,10 +425,9 @@ class SecretCacheEntry : public CInterface friend class SecretCache; public: - //A cache entry is initally created that has a create and access time of now, but the checkTimestamp - //is set so that needsRefresh() will return true. + //A cache entry is initally created that has a create and access,and check time of now SecretCacheEntry(cache_timestamp _now, const char * _secretKey) - : secretKey(_secretKey), contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now - 2 * secretTimeoutNs) + : secretKey(_secretKey), contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now) { } @@ -530,9 +529,10 @@ class SecretCache } //Check to see if a secret exists, and if not add a null entry that has expired. - SecretCacheEntry * getSecret(const std::string & secretKey, cache_timestamp now) + SecretCacheEntry * getSecret(const std::string & secretKey, cache_timestamp now, bool & isNewEntry) { SecretCacheEntry * result; + isNewEntry = false; CriticalBlock block(cs); auto match = secrets.find(secretKey); if (match != secrets.cend()) @@ -542,9 +542,10 @@ class SecretCache } else { - //Insert an entry with a null value that is marked as out of date + //Insert an entry with a null value result = new SecretCacheEntry(now, secretKey.c_str()); secrets.emplace(secretKey, result); + isNewEntry = true; } return result; } @@ -1170,8 +1171,9 @@ static SecretCacheEntry * getSecretEntry(const char * category, const char * nam std::string key(buildSecretKey(category, name, optVaultId, optVersion)); - SecretCacheEntry * match = globalSecretCache.getSecret(key, now); - if (!match->needsRefresh(now)) + bool isNewEntry; + SecretCacheEntry * match = globalSecretCache.getSecret(key, now, isNewEntry); + if (!isNewEntry && !match->needsRefresh(now)) return match; Owned resolved(resolveSecret(category, name, optVaultId, optVersion)); @@ -1314,7 +1316,7 @@ void CSecret::checkUptoDate() const if (secret->needsRefresh(now)) { #ifdef TRACE_SECRETS - DBGLOG("Secret %s is stale updating from %u...", secret->queryTraceName(), secretHash); + DBGLOG("Secret %s is stale updating...", secret->queryTraceName()); #endif //MORE: This could block or fail - in roxie especially it would be better to return the old value try diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index ba972fc220a..e8001c068ed 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -4252,8 +4252,8 @@ class JLibSecretsTest : public CppUnit::TestFixture CPPUNIT_ASSERT(!secret6->isValid()); CPPUNIT_ASSERT(!secret6->isStale()); - //Sleep so the cache entry should have expired and the value reread since reading ahead - MilliSleep(60); // elapsed=110 = 80 + 30 + //Sleep so the cache entry should have expired (between 80 and 85ms) and the value reread since reading ahead + MilliSleep(60); // elapsed=110 = 50 + 60 CPPUNIT_ASSERT(secret6->isValid()); CPPUNIT_ASSERT(!secret6->isStale()); unsigned version1 = secret6->getVersion(); // Mark the value as accessed, but too early to be refreshed From 02b78f41d12f767331ef52646d2149fd0e064d31 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 19 Mar 2024 17:22:30 +0000 Subject: [PATCH 13/16] HPCC-31494 Avoid spurious follow-on 'failed to run' error Signed-off-by: Gavin Halliday --- ecl/eclagent/eclagentmain.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ecl/eclagent/eclagentmain.cpp b/ecl/eclagent/eclagentmain.cpp index dcd04d6152a..3e305eeec09 100644 --- a/ecl/eclagent/eclagentmain.cpp +++ b/ecl/eclagent/eclagentmain.cpp @@ -59,6 +59,9 @@ int main(int argc, const char *argv[]) try { ret = eclagent_main(argc, argv); + //Do not return a non-zero error code in containerized mode - otherwise the system will think it failed to run + if (isContainerized()) + ret = 0; } catch (IException *E) { From 5c0588f3257d26cd3661ef3d3192ddbe3ce69a8a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 15 Mar 2024 11:13:38 +0000 Subject: [PATCH 14/16] HPCC-31465 Update system to use the most recent open telemetry libraries Signed-off-by: Gavin Halliday --- system/jlib/jtrace.cpp | 9 +- .../add-missing-dependencies.patch | 15 ---- .../add-missing-find-dependency.patch | 13 --- .../hpcc-remove-unsafe-onfork.patch | 56 ------------ .../opentelemetry-cpp/portfile.cmake | 75 ---------------- .../use-default-cxx-version.patch | 26 ------ vcpkg_overlays/opentelemetry-cpp/vcpkg.json | 87 ------------------- 7 files changed, 7 insertions(+), 274 deletions(-) delete mode 100644 vcpkg_overlays/opentelemetry-cpp/add-missing-dependencies.patch delete mode 100644 vcpkg_overlays/opentelemetry-cpp/add-missing-find-dependency.patch delete mode 100644 vcpkg_overlays/opentelemetry-cpp/hpcc-remove-unsafe-onfork.patch delete mode 100644 vcpkg_overlays/opentelemetry-cpp/portfile.cmake delete mode 100644 vcpkg_overlays/opentelemetry-cpp/use-default-cxx-version.patch delete mode 100644 vcpkg_overlays/opentelemetry-cpp/vcpkg.json diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 04bdaf19e98..e5ab955c533 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -17,7 +17,9 @@ #include "opentelemetry/trace/semantic_conventions.h" //known span defines #include "opentelemetry/context/propagation/global_propagator.h" // context::propagation::GlobalTextMapPropagator::GetGlobalPropagator +#include "opentelemetry/sdk/trace/batch_span_processor_options.h" //opentelemetry::sdk::trace::TracerProviderFactory::Create(context) #include "opentelemetry/sdk/trace/tracer_provider_factory.h" //opentelemetry::sdk::trace::TracerProviderFactory::Create(context) +#include "opentelemetry/sdk/trace/tracer_context.h" //opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); #include "opentelemetry/sdk/trace/tracer_context_factory.h" //opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors)); #include "opentelemetry/sdk/trace/simple_processor_factory.h" #include "opentelemetry/sdk/trace/batch_span_processor_factory.h" @@ -48,6 +50,9 @@ #include "lnuid.h" #include +//This seems to be defined in some window builds - avoid conflicts with the functions below +#undef max + namespace context = opentelemetry::context; namespace nostd = opentelemetry::nostd; namespace opentel_trace = opentelemetry::trace; @@ -1321,10 +1326,10 @@ void CTraceManager::initTracerProviderAndGlobalInternals(const IPropertyTree * t auto jtraceResource = opentelemetry::sdk::resource::Resource::Create(resourceAtts); // Default is an always-on sampler. - std::shared_ptr context = + std::unique_ptr context = opentelemetry::sdk::trace::TracerContextFactory::Create(std::move(processors), jtraceResource); std::shared_ptr provider = - opentelemetry::sdk::trace::TracerProviderFactory::Create(context); + opentelemetry::sdk::trace::TracerProviderFactory::Create(std::move(context)); // Set the global trace provider opentelemetry::trace::Provider::SetTracerProvider(provider); diff --git a/vcpkg_overlays/opentelemetry-cpp/add-missing-dependencies.patch b/vcpkg_overlays/opentelemetry-cpp/add-missing-dependencies.patch deleted file mode 100644 index 2de8be70c7a..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/add-missing-dependencies.patch +++ /dev/null @@ -1,15 +0,0 @@ -diff --git a/cmake/opentelemetry-proto.cmake b/cmake/opentelemetry-proto.cmake -index 34b33d3..19e67e9 100644 ---- a/cmake/opentelemetry-proto.cmake -+++ b/cmake/opentelemetry-proto.cmake -@@ -311,6 +311,10 @@ if(WITH_OTLP_GRPC) - endif() - endif() - -+if(TARGET gRPC::grpc++) -+ target_link_libraries(opentelemetry_proto PUBLIC gRPC::grpc++) -+endif() -+ - if(BUILD_SHARED_LIBS) - foreach(proto_target ${OPENTELEMETRY_PROTO_TARGETS}) - set_property(TARGET ${proto_target} PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/vcpkg_overlays/opentelemetry-cpp/add-missing-find-dependency.patch b/vcpkg_overlays/opentelemetry-cpp/add-missing-find-dependency.patch deleted file mode 100644 index 1f9c12d1636..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/add-missing-find-dependency.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/cmake/opentelemetry-cpp-config.cmake.in b/cmake/opentelemetry-cpp-config.cmake.in -index adae58d..2642772 100644 ---- a/cmake/opentelemetry-cpp-config.cmake.in -+++ b/cmake/opentelemetry-cpp-config.cmake.in -@@ -69,6 +69,8 @@ set(OPENTELEMETRY_VERSION - # ############################################################################## - - find_package(Threads) -+include(CMakeFindDependencyMacro) -+find_dependency(absl) - - set_and_check(OPENTELEMETRY_CPP_INCLUDE_DIRS "@PACKAGE_INCLUDE_INSTALL_DIR@") - set_and_check(OPENTELEMETRY_CPP_LIBRARY_DIRS "@PACKAGE_CMAKE_INSTALL_LIBDIR@") diff --git a/vcpkg_overlays/opentelemetry-cpp/hpcc-remove-unsafe-onfork.patch b/vcpkg_overlays/opentelemetry-cpp/hpcc-remove-unsafe-onfork.patch deleted file mode 100644 index b9a0a16725a..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/hpcc-remove-unsafe-onfork.patch +++ /dev/null @@ -1,56 +0,0 @@ -diff --git a/sdk/src/common/random.cc b/sdk/src/common/random.cc -index 77b88cfa..14e94d0c 100644 ---- a/sdk/src/common/random.cc -+++ b/sdk/src/common/random.cc -@@ -13,11 +13,10 @@ namespace sdk - { - namespace common - { --// Wraps a thread_local random number generator, but adds a fork handler so that --// the generator will be correctly seeded after forking. --// --// See https://stackoverflow.com/q/51882689/4447365 and --// https://github.com/opentracing-contrib/nginx-opentracing/issues/52 -+// Wraps a thread_local random number generator. -+// The previous fork handler is removed because it was not safe and was solving a problem that did -+// not need to be solved since there should be no logic in the child() before it calls exec(). -+ - namespace - { - class TlsRandomNumberGenerator -@@ -26,17 +25,14 @@ class TlsRandomNumberGenerator - TlsRandomNumberGenerator() noexcept - { - Seed(); -- platform::AtFork(nullptr, nullptr, OnFork); - } - -- static FastRandomNumberGenerator &engine() noexcept { return engine_; } -+ FastRandomNumberGenerator & engine() noexcept { return engine_; } - - private: -- static thread_local FastRandomNumberGenerator engine_; -- -- static void OnFork() noexcept { Seed(); } -+ FastRandomNumberGenerator engine_; - -- static void Seed() noexcept -+ void Seed() noexcept - { - std::random_device random_device; - std::seed_seq seed_seq{random_device(), random_device(), random_device(), random_device()}; -@@ -44,13 +40,12 @@ class TlsRandomNumberGenerator - } - }; - --thread_local FastRandomNumberGenerator TlsRandomNumberGenerator::engine_{}; - } // namespace - - FastRandomNumberGenerator &Random::GetRandomNumberGenerator() noexcept - { - static thread_local TlsRandomNumberGenerator random_number_generator{}; -- return TlsRandomNumberGenerator::engine(); -+ return random_number_generator.engine(); - } - - uint64_t Random::GenerateRandom64() noexcept diff --git a/vcpkg_overlays/opentelemetry-cpp/portfile.cmake b/vcpkg_overlays/opentelemetry-cpp/portfile.cmake deleted file mode 100644 index facb6883226..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/portfile.cmake +++ /dev/null @@ -1,75 +0,0 @@ -if(VCPKG_TARGET_IS_WINDOWS) - vcpkg_check_linkage(ONLY_STATIC_LIBRARY) -endif() - -vcpkg_from_github( - OUT_SOURCE_PATH SOURCE_PATH - REPO open-telemetry/opentelemetry-cpp - REF "v${VERSION}" - SHA512 86cf0320f9ee50bc1aa2b7a8b254fb0df25d1bd1f5f01ebc3630ab7fe2f6ca5e53ca8e042518b4e7096dbb102c0b880e9a25fcdf5f668d24ff57d9247237bf62 - HEAD_REF main - PATCHES - # Use the compiler's default C++ version. Picking a version with - # CMAKE_CXX_STANDARD is not needed as the Abseil port already picked - # one and propagates that version across all its downstream deps. - use-default-cxx-version.patch - # When compiling code generated by gRPC we need to link the gRPC library - # too. - add-missing-dependencies.patch - # Missing find_dependency for Abseil - add-missing-find-dependency.patch - # HPCC-fix: Remove code that reinitialised the random number generator on fork() - hpcc-remove-unsafe-onfork.patch -) - -vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS - FEATURES - etw WITH_ETW - zipkin WITH_ZIPKIN - prometheus WITH_PROMETHEUS - elasticsearch WITH_ELASTICSEARCH - jaeger WITH_JAEGER - otlp WITH_OTLP - otlp-http WITH_OTLP_HTTP - zpages WITH_ZPAGES - otlp-grpc WITH_OTLP_GRPC -) - -# opentelemetry-proto is a third party submodule and opentelemetry-cpp release did not pack it. -if(WITH_OTLP) - set(OTEL_PROTO_VERSION "0.19.0") - vcpkg_download_distfile(ARCHIVE - URLS "https://github.com/open-telemetry/opentelemetry-proto/archive/v${OTEL_PROTO_VERSION}.tar.gz" - FILENAME "opentelemetry-proto-${OTEL_PROTO_VERSION}.tar.gz" - SHA512 b6d47aaa90ff934eb24047757d5fdb8a5be62963a49b632460511155f09a725937fb7535cf34f738b81cc799600adbbc3809442aba584d760891c0a1f0ce8c03 - ) - - vcpkg_extract_source_archive(src ARCHIVE "${ARCHIVE}") - file(REMOVE_RECURSE "${SOURCE_PATH}/third_party/opentelemetry-proto") - file(COPY "${src}/." DESTINATION "${SOURCE_PATH}/third_party/opentelemetry-proto") - # Create empty .git directory to prevent opentelemetry from cloning it during build time - file(MAKE_DIRECTORY "${SOURCE_PATH}/third_party/opentelemetry-proto/.git") - list(APPEND FEATURE_OPTIONS -DCMAKE_CXX_STANDARD=14) - list(APPEND FEATURE_OPTIONS -DgRPC_CPP_PLUGIN_EXECUTABLE=${CURRENT_HOST_INSTALLED_DIR}/tools/grpc/grpc_cpp_plugin${VCPKG_HOST_EXECUTABLE_SUFFIX}) -endif() - -vcpkg_cmake_configure( - SOURCE_PATH "${SOURCE_PATH}" - OPTIONS - -DBUILD_TESTING=OFF - -DWITH_EXAMPLES=OFF - -DWITH_LOGS_PREVIEW=ON - -DOPENTELEMETRY_INSTALL=ON - -DWITH_ABSEIL=ON - ${FEATURE_OPTIONS} - MAYBE_UNUSED_VARIABLES - WITH_OTLP_GRPC -) - -vcpkg_cmake_install() -vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${PORT}) -vcpkg_copy_pdbs() - -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") -file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") -vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/vcpkg_overlays/opentelemetry-cpp/use-default-cxx-version.patch b/vcpkg_overlays/opentelemetry-cpp/use-default-cxx-version.patch deleted file mode 100644 index 53a31faf91b..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/use-default-cxx-version.patch +++ /dev/null @@ -1,26 +0,0 @@ -diff --git a/CMakeLists.txt b/CMakeLists.txt -index f4fa064..a868106 100644 ---- a/CMakeLists.txt -+++ b/CMakeLists.txt -@@ -126,21 +126,6 @@ endif() - option(OPENTELEMETRY_INSTALL "Whether to install opentelemetry targets" - ${OPENTELEMETRY_INSTALL_default}) - --if(NOT DEFINED CMAKE_CXX_STANDARD) -- if(WITH_STL) -- # Require at least C++17. C++20 is needed to avoid gsl::span -- if(CMAKE_VERSION VERSION_GREATER 3.11.999) -- # Ask for 20, may get anything below -- set(CMAKE_CXX_STANDARD 20) -- else() -- # Ask for 17, may get anything below -- set(CMAKE_CXX_STANDARD 17) -- endif() -- else() -- set(CMAKE_CXX_STANDARD 11) -- endif() --endif() -- - if(WITH_STL) - # These definitions are needed for test projects that do not link against - # opentelemetry-api library directly. We ensure that variant implementation diff --git a/vcpkg_overlays/opentelemetry-cpp/vcpkg.json b/vcpkg_overlays/opentelemetry-cpp/vcpkg.json deleted file mode 100644 index b525fbeb2d5..00000000000 --- a/vcpkg_overlays/opentelemetry-cpp/vcpkg.json +++ /dev/null @@ -1,87 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", - "name": "opentelemetry-cpp", - "version-semver": "1.9.1", - "description": [ - "OpenTelemetry is a collection of tools, APIs, and SDKs.", - "You use it to instrument, generate, collect, and export telemetry data (metrics, logs, and traces) for analysis in order to understand your software's performance and behavior." - ], - "homepage": "https://github.com/open-telemetry/opentelemetry-cpp", - "license": "Apache-2.0", - "dependencies": [ - "abseil", - "curl", - "nlohmann-json", - { - "name": "vcpkg-cmake", - "host": true - }, - { - "name": "vcpkg-cmake-config", - "host": true - } - ], - "features": { - "elasticsearch": { - "description": "Whether to include the Elasticsearch Client in the SDK" - }, - "etw": { - "description": "Whether to include the ETW Exporter in the SDK", - "supports": "windows" - }, - "jaeger": { - "description": "Whether to include the Jaeger exporter", - "dependencies": [ - "thrift" - ] - }, - "otlp": { - "description": "Whether to include the OpenTelemetry Protocol in the SDK", - "dependencies": [ - "protobuf" - ] - }, - "otlp-grpc": { - "description": "Whether to include the OTLP gRPC exporter in the SDK", - "dependencies": [ - "grpc", - { - "name": "grpc", - "host": true - }, - { - "name": "opentelemetry-cpp", - "default-features": false, - "features": [ - "otlp" - ] - } - ] - }, - "otlp-http": { - "description": "Whether to include the OpenTelemetry Protocol over HTTP in the SDK", - "dependencies": [ - "curl", - { - "name": "opentelemetry-cpp", - "default-features": false, - "features": [ - "otlp" - ] - } - ] - }, - "prometheus": { - "description": "Whether to include the Prometheus Client in the SDK", - "dependencies": [ - "prometheus-cpp" - ] - }, - "zipkin": { - "description": "Whether to include the Zipkin exporter in the SDK" - }, - "zpages": { - "description": "Whether to include the Zpages Server in the SDK" - } - } -} From 2f3546f11a740e841efa307b131e3db85620552d Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 20 Mar 2024 09:46:41 +0000 Subject: [PATCH 15/16] HPCC-31497 Improve error details if unexpected node type read from index Signed-off-by: Gavin Halliday --- system/jhtree/jhtree.cpp | 11 +++++++++-- system/jhtree/jhtree.hpp | 2 +- system/jhtree/jhtree.ipp | 7 +++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index be4413d4356..084d0e9bff4 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -2441,7 +2441,7 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface virtual IKeyIndex *queryPart(unsigned idx) { return idx ? NULL : this; } virtual unsigned queryScans() { return realKey ? realKey->queryScans() : 0; } virtual unsigned querySeeks() { return realKey ? realKey->querySeeks() : 0; } - virtual const char *queryFileName() { return keyfile.get(); } + virtual const char *queryFileName() const { return keyfile.get(); } virtual offset_t queryBlobHead() { return checkOpen().queryBlobHead(); } virtual void resetCounts() { if (realKey) realKey->resetCounts(); } virtual offset_t queryLatestGetNodeOffset() const { return realKey ? realKey->queryLatestGetNodeOffset() : 0; } @@ -2659,7 +2659,14 @@ const CJHTreeNode *CNodeCache::getNode(const INodeLoader *keyIndex, unsigned iD, if (!ownedCacheEntry->isReady()) { const CJHTreeNode *node = keyIndex->loadNode(&fetchCycles, pos); - assertex(type == node->getNodeType()); + if (unlikely(type != node->getNodeType())) + { + //This should never happen, but if it does, report as much information as possible to diagnose the issue. + StringBuffer msg; + msg.appendf("Node type mismatch for node %s@%llx (expected %s, got %s)", keyIndex->queryFileName(), pos, cacheTypeText[type], cacheTypeText[node->getNodeType()]); + node->Release(); + throwUnexpectedX(msg); + } //Update the associated size of the entry in the hash table before setting isReady (never evicted until isReady is set) curCache.noteReady(*node); diff --git a/system/jhtree/jhtree.hpp b/system/jhtree/jhtree.hpp index d22531c3b70..8879696ef59 100644 --- a/system/jhtree/jhtree.hpp +++ b/system/jhtree/jhtree.hpp @@ -90,7 +90,7 @@ interface jhtree_decl IKeyIndex : public IKeyIndexBase virtual unsigned querySeeks() = 0; virtual size32_t keyedSize() = 0; virtual bool hasPayload() = 0; - virtual const char *queryFileName() = 0; + virtual const char *queryFileName() const = 0; virtual offset_t queryBlobHead() = 0; virtual void resetCounts() = 0; virtual offset_t queryLatestGetNodeOffset() const = 0; diff --git a/system/jhtree/jhtree.ipp b/system/jhtree/jhtree.ipp index 9280e967f40..93ae8013db7 100644 --- a/system/jhtree/jhtree.ipp +++ b/system/jhtree/jhtree.ipp @@ -78,6 +78,7 @@ interface INodeLoader virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const = 0; virtual const CJHSearchNode *locateFirstLeafNode(IContextLogger *ctx) const = 0; virtual const CJHSearchNode *locateLastLeafNode(IContextLogger *ctx) const = 0; + virtual const char *queryFileName() const = 0; }; class jhtree_decl CKeyIndex : implements IKeyIndex, implements INodeLoader, public CInterface @@ -161,6 +162,8 @@ public: virtual const CJHSearchNode *locateLastLeafNode(IContextLogger *ctx) const override; virtual void mergeStats(CRuntimeStatisticCollection & stats) const override {} + + virtual const char *queryFileName() const = 0; }; class jhtree_decl CMemKeyIndex : public CKeyIndex @@ -170,7 +173,7 @@ private: public: CMemKeyIndex(unsigned _iD, IMemoryMappedFile *_io, const char *_name, bool _isTLK); - virtual const char *queryFileName() { return name.get(); } + virtual const char *queryFileName() const { return name.get(); } virtual const IFileIO *queryFileIO() const override { return nullptr; } // INodeLoader impl. virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const override; @@ -186,7 +189,7 @@ private: public: CDiskKeyIndex(unsigned _iD, IFileIO *_io, const char *_name, bool _isTLK); - virtual const char *queryFileName() { return name.get(); } + virtual const char *queryFileName() const { return name.get(); } virtual const IFileIO *queryFileIO() const override { return io; } // INodeLoader impl. virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const override; From e2424b2e013bee2f705eb20b303e7cee95c9b09b Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 21 Mar 2024 16:10:19 +0000 Subject: [PATCH 16/16] HPCC-31468 Prevent propagation of AKS remote file meta info When a k8s logical file's meta data is retrieved, it is supplemented with info. about the remote plane characteristics the remote file resides on. This is used by the clients DFS to know how to map file into in lookup calls to the k8s dafilesrv service. The logical file meta data as a whole can be stored as extra meta data by the roxie file deploy mechanism, which is then used as a source when Roxie is starting, to copy locally. If a secondary Roxie uses a Roxie which copied from a k8s, it will also see and use this remote plane info spuriously, and then use it to manipulate the group/file paths, to point to the original k8s source. Suppress this info. being propagated. Signed-off-by: Jake Smith --- dali/base/dadfs.cpp | 29 ++++++++++++++--------------- dali/dfu/dfuutil.cpp | 23 +++++++++++++---------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 6e2c6b5f113..17d9dd0a51d 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -11215,23 +11215,22 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I Owned tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false); if (tree) { - if (isContainerized()) + // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup, + // asking for the returned meta data to be remapped to point to the dafilesrv service. + if (isContainerized() && hasMask(opts, GetFileTreeOpts::remapToService)) { - // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup, - // asking for the returned meta data to be remapped to point to the dafilesrv service. - if (hasMask(opts, GetFileTreeOpts::remapToService)) - { - tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv - remapGroupsToDafilesrv(tree, true, secureService); - groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) - - const char *remotePlaneName = tree->queryProp("@group"); - Owned filePlane = getStoragePlane(remotePlaneName); - assertex(filePlane); - // Used by DFS clients to determine if stripe and/or alias translation needed - tree->setPropTree("Attr/_remoteStoragePlane", createPTreeFromIPT(filePlane)); - } + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv + remapGroupsToDafilesrv(tree, true, secureService); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) + + const char *remotePlaneName = tree->queryProp("@group"); + Owned filePlane = getStoragePlane(remotePlaneName); + assertex(filePlane); + // Used by DFS clients to determine if stripe and/or alias translation needed + tree->setPropTree("Attr/_remoteStoragePlane", createPTreeFromIPT(filePlane)); } + else + tree->removeProp("Attr/_remoteStoragePlane"); Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)1); // 1 == standard file diff --git a/dali/dfu/dfuutil.cpp b/dali/dfu/dfuutil.cpp index 8ec3f62b07c..c1ae5867a41 100644 --- a/dali/dfu/dfuutil.cpp +++ b/dali/dfu/dfuutil.cpp @@ -419,17 +419,20 @@ class CFileCloner // for now, only use source file descriptor as cloned source if it's from // wsdfs file backed by remote storage using dafilesrv (NB: if it is '_remoteStoragePlane' will be set) // JCSMORE: it may be this can replace the need for the other 'clone*' attributes altogether. - if (srcfdesc->queryProperties().hasProp("_remoteStoragePlane")) + if (srcfdesc->queryProperties().hasProp("_remoteStoragePlane") && srcdali && !srcdali->endpoint().isNull()) { - if (srcdali && !srcdali->endpoint().isNull()) - { - attrs.setPropTree("cloneFromFDesc", createPTreeFromIPT(srcTree)); - StringBuffer host; - attrs.setProp("@cloneFrom", srcdali->endpoint().getEndpointHostText(host).str()); - if (prefix.length()) - attrs.setProp("@cloneFromPrefix", prefix.get()); - return; - } + attrs.setPropTree("cloneFromFDesc", createPTreeFromIPT(srcTree)); + StringBuffer host; + attrs.setProp("@cloneFrom", srcdali->endpoint().getEndpointHostText(host).str()); + if (prefix.length()) + attrs.setProp("@cloneFromPrefix", prefix.get()); + return; + } + else + { + attrs.removeProp("cloneFromFDesc"); + attrs.removeProp("@cloneFrom"); + attrs.removeProp("@cloneFromPrefix"); } while(attrs.removeProp("cloneFromGroup"));