diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 1ff1c0fee5d..030af8e53ca 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -11215,23 +11215,22 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I Owned tree = getNamedPropTree(sroot,queryDfsXmlBranchName(DXB_File),"@name",tail.str(),false); if (tree) { - if (isContainerized()) + // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup, + // asking for the returned meta data to be remapped to point to the dafilesrv service. + if (isContainerized() && hasMask(opts, GetFileTreeOpts::remapToService)) { - // This is for bare-metal clients using ~foreign pointing at a containerized/k8s setup, - // asking for the returned meta data to be remapped to point to the dafilesrv service. - if (hasMask(opts, GetFileTreeOpts::remapToService)) - { - tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv - remapGroupsToDafilesrv(tree, true, secureService); - groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) - - const char *remotePlaneName = tree->queryProp("@group"); - Owned filePlane = getStoragePlane(remotePlaneName); - assertex(filePlane); - // Used by DFS clients to determine if stripe and/or alias translation needed - tree->setPropTree("Attr/_remoteStoragePlane", createPTreeFromIPT(filePlane)); - } + tree.setown(createPTreeFromIPT(tree)); // copy live Dali tree, because it is about to be altered by remapGroupsToDafilesrv + remapGroupsToDafilesrv(tree, true, secureService); + groupResolver = nullptr; // do not attempt to resolve remapped group (it will not exist and cause addUnique to create a new anon one) + + const char *remotePlaneName = tree->queryProp("@group"); + Owned filePlane = getStoragePlane(remotePlaneName); + assertex(filePlane); + // Used by DFS clients to determine if stripe and/or alias translation needed + tree->setPropTree("Attr/_remoteStoragePlane", createPTreeFromIPT(filePlane)); } + else + tree->removeProp("Attr/_remoteStoragePlane"); Owned fdesc = deserializeFileDescriptorTree(tree,groupResolver,IFDSF_EXCLUDE_CLUSTERNAMES); mb.append((int)1); // 1 == standard file diff --git a/dali/dfu/dfuutil.cpp b/dali/dfu/dfuutil.cpp index 8ec3f62b07c..c1ae5867a41 100644 --- a/dali/dfu/dfuutil.cpp +++ b/dali/dfu/dfuutil.cpp @@ -419,17 +419,20 @@ class CFileCloner // for now, only use source file descriptor as cloned source if it's from // wsdfs file backed by remote storage using dafilesrv (NB: if it is '_remoteStoragePlane' will be set) // JCSMORE: it may be this can replace the need for the other 'clone*' attributes altogether. - if (srcfdesc->queryProperties().hasProp("_remoteStoragePlane")) + if (srcfdesc->queryProperties().hasProp("_remoteStoragePlane") && srcdali && !srcdali->endpoint().isNull()) { - if (srcdali && !srcdali->endpoint().isNull()) - { - attrs.setPropTree("cloneFromFDesc", createPTreeFromIPT(srcTree)); - StringBuffer host; - attrs.setProp("@cloneFrom", srcdali->endpoint().getEndpointHostText(host).str()); - if (prefix.length()) - attrs.setProp("@cloneFromPrefix", prefix.get()); - return; - } + attrs.setPropTree("cloneFromFDesc", createPTreeFromIPT(srcTree)); + StringBuffer host; + attrs.setProp("@cloneFrom", srcdali->endpoint().getEndpointHostText(host).str()); + if (prefix.length()) + attrs.setProp("@cloneFromPrefix", prefix.get()); + return; + } + else + { + attrs.removeProp("cloneFromFDesc"); + attrs.removeProp("@cloneFrom"); + attrs.removeProp("@cloneFromPrefix"); } while(attrs.removeProp("cloneFromGroup")); diff --git a/ecl/eclagent/eclagentmain.cpp b/ecl/eclagent/eclagentmain.cpp index dcd04d6152a..3e305eeec09 100644 --- a/ecl/eclagent/eclagentmain.cpp +++ b/ecl/eclagent/eclagentmain.cpp @@ -59,6 +59,9 @@ int main(int argc, const char *argv[]) try { ret = eclagent_main(argc, argv); + //Do not return a non-zero error code in containerized mode - otherwise the system will think it failed to run + if (isContainerized()) + ret = 0; } catch (IException *E) { diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 92e230cfd67..ca4ee1de6ae 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -1,6 +1,7 @@ import * as React from "react"; import { CommandBar, ContextualMenuItemType, ICommandBarItemProps } from "@fluentui/react"; -import { WUDetails, IScope } from "@hpcc-js/comms"; +import { Workunit, WUDetails, IScope } from "@hpcc-js/comms"; +import { scopedLogger } from "@hpcc-js/util"; import nlsHPCC from "src/nlsHPCC"; import { useWorkunitArchive } from "../hooks/workunit"; import { useWorkunitMetrics } from "../hooks/metrics"; @@ -12,6 +13,8 @@ import { ECLArchiveTree } from "./ECLArchiveTree"; import { ECLArchiveEditor } from "./ECLArchiveEditor"; import { MetricsPropertiesTables } from "./MetricsPropertiesTables"; +const logger = scopedLogger("src-react/components/ECLArchive.tsx"); + const scopeFilterDefault: WUDetails.RequestNS.ScopeFilter = { MaxDepth: 999999, ScopeTypes: ["graph"] @@ -54,8 +57,15 @@ export const ECLArchive: React.FunctionComponent = ({ setSelectionText(archive?.content(selection) ?? ""); setMarkers(archive?.markers(selection) ?? []); setSelectedMetrics(archive?.metrics(selection) ?? []); + } else { + if (archive && !archive.build) { + const wu = Workunit.attach({ baseUrl: "" }, wuid); + wu.fetchQuery().then(function (query) { + setSelectionText(query?.Text ?? ""); + }).catch(err => logger.error(err)); + } } - }, [archive, metrics.length, selection]); + }, [archive, metrics.length, selection, wuid]); const setSelectedItem = React.useCallback((selId: string) => { pushUrl(`${parentUrl}/${selId}`); diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index 16bdf49b754..9321582024d 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -136,8 +136,8 @@ const AsyncDropdown: React.FunctionComponent = ({ }, [onChange, selectedItem, selectedIdx, selectedKey]); return options === undefined ? - : - setSelectedItem(item)} placeholder={placeholder} disabled={disabled} required={required} errorMessage={errorMessage} className={className} />; + : + setSelectedItem(item)} placeholder={placeholder} disabled={disabled} required={required} errorMessage={errorMessage} className={className} />; }; interface DropdownMultiProps { diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 560685e03f2..207f0dd5a52 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -1235,11 +1235,14 @@ class CRemoteDiskBaseActivity : public CSimpleInterfaceOf, } virtual void serializeCursor(MemoryBuffer &tgt) const override { - throwUnexpected(); + // we need to serialize something, because the lack of a cursor is used to signify end of stream + // NB: the cursor is opaque and only to be consumed by dafilesrv. When used it is simply passed back. + tgt.append("UNSUPPORTED"); } virtual void restoreCursor(MemoryBuffer &src) override { - throwUnexpected(); + throw makeStringExceptionV(0, "restoreCursor not supported in: %s", typeid(*this).name()); + throwUnimplemented(); } virtual void flushStatistics(CClientStats &stats) override { @@ -2019,11 +2022,13 @@ class CRemoteJsonReadActivity : public CRemoteMarkupReadActivity public: CRemoteJsonReadActivity(IPropertyTree &config, IFileDescriptor *fileDesc) : PARENT(config, fileDesc, TAKjsonread) { - xpath.set("/"); if (customRowTag.isEmpty()) // no override fileDesc->queryProperties().getProp("@rowTag", xpath); else + { + xpath.set("/"); xpath.append(customRowTag); + } } }; @@ -2392,11 +2397,13 @@ class CRemoteWriteBaseActivity : public CSimpleInterfaceOf } virtual void serializeCursor(MemoryBuffer &tgt) const override { - throwUnexpected(); + // we need to serialize something, because the lack of a cursor is used to signify end of stream + // NB: the cursor is opaque and only to be consumed by dafilesrv. When used it is simply passed back. + tgt.append("UNSUPPORTED"); } virtual void restoreCursor(MemoryBuffer &src) override { - throwUnexpected(); + throw makeStringExceptionV(0, "restoreCursor not supported in: %s", typeid(*this).name()); } virtual StringBuffer &getInfoStr(StringBuffer &out) const override { diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index 86a64edd0f3..214e3233330 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -483,7 +483,11 @@ inline unsigned getBondedChannel(unsigned partNo) extern void FatalError(const char *format, ...) __attribute__((format(printf, 1, 2))); extern unsigned getNextInstanceId(); extern void closedown(); -extern void saveTopology(); +extern void saveTopology(bool lockDali); +extern unsigned __int64 getTopologyHash(); + +extern unsigned __int64 currentTopologyHash; +extern unsigned __int64 originalTopologyHash; #define LOGGING_INTERCEPTED 0x01 #define LOGGING_TIMEACTIVITIES 0x02 diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 5cca34e0519..5490b81b7fe 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -419,16 +419,18 @@ int myhook(int alloctype, void *, size_t nSize, int p1, long allocSeq, const uns } #endif -void saveTopology() +void saveTopology(bool lockDali) { - // Write back changes that have been made via certain control:xxx changes, so that they survive a roxie restart + // Write back changes that have been made via control:(un)lockDali changes, so that they survive a roxie restart // Note that they are overwritten when Roxie is manually stopped/started via hpcc-init service - these changes // are only intended to be temporary for the current session if (!useOldTopology) return; try { - saveXML(topologyFile.str(), topology); + Owned tempTopology = createPTreeFromXMLFile(topologyFile.str(), ipt_caseInsensitive); + tempTopology->setPropBool("@lockDali", lockDali); + saveXML(topologyFile.str(), tempTopology); } catch (IException *E) { @@ -461,6 +463,16 @@ static std::vector> agentChannels; void *leakChecker = nullptr; // Used to deliberately leak an allocation to ensure leak checking is working +unsigned __int64 currentTopologyHash = 0; +unsigned __int64 originalTopologyHash = 0; + +hash64_t getTopologyHash() +{ + StringBuffer xml; + toXML(topology, xml, 0, XML_SortTags); + return rtlHash64Data(xml.length(), xml.str(), 707018); +} + #ifndef _CONTAINERIZED void readStaticTopology() { @@ -695,7 +707,8 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) topologyFile.append(codeDirectory).append(PATHSEPCHAR).append("RoxieTopology.xml"); useOldTopology = checkFileExists(topologyFile.str()); topology = loadConfiguration(useOldTopology ? nullptr : defaultYaml, argv, "roxie", "ROXIE", topologyFile, nullptr, "@netAddress"); - saveTopology(); + saveTopology(topology->getPropBool("@lockDali", false)); + originalTopologyHash = currentTopologyHash = getTopologyHash(); // Any settings we read from topology that must NOT be overridden in workunit debug fields should be read at this point, before the following section getAllowedPipePrograms(allowedPipePrograms, true); diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index 37c3496e307..5e41da7fbbf 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -1041,6 +1041,8 @@ struct PingRecord { unsigned tick; IpAddress senderIP; + unsigned __int64 currentTopoHash; + unsigned __int64 originalTopoHash; }; void doPing(IRoxieQueryPacket *packet, const IRoxieContextLogger &logctx) @@ -1059,6 +1061,16 @@ void doPing(IRoxieQueryPacket *packet, const IRoxieContextLogger &logctx) StringBuffer s; throw MakeStringException(ROXIE_UNKNOWN_SERVER, "Message received from unknown Roxie server %s", header.toString(s).str()); } + if (originalTopologyHash != data->originalTopoHash) + { + StringBuffer s; + EXCLOG(MCoperatorError,"ERROR: Configuration file mismatch detected with Roxie server %s", header.toString(s).str()); + } + if (currentTopologyHash != data->currentTopoHash) + { + StringBuffer s; + DBGLOG("WARNING: Temporary configuration mismatch detected with Roxie server %s", header.toString(s).str()); + } RoxiePacketHeader newHeader(header, ROXIE_PING, 0); // subchannel not relevant Owned output = ROQ->createOutputStream(newHeader, true, logctx); void *ret = output->getBuffer(contextLength, false); @@ -3887,6 +3899,8 @@ class PingTimer : public Thread PingRecord data; data.senderIP.ipset(myNode.getIpAddress()); data.tick = usTick(); + data.originalTopoHash = originalTopologyHash; + data.currentTopoHash = currentTopologyHash; mb.append(sizeof(PingRecord), &data); if (doTrace(traceRoxiePings)) DBGLOG("PING sent"); diff --git a/roxie/ccd/ccdstate.cpp b/roxie/ccd/ccdstate.cpp index 6082592361b..c09c603e6ed 100644 --- a/roxie/ccd/ccdstate.cpp +++ b/roxie/ccd/ccdstate.cpp @@ -2580,7 +2580,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme topology->setPropBool("@lockDali", true); if (daliHelper) daliHelper->disconnect(); - saveTopology(); + saveTopology(true); } else if (stricmp(queryName, "control:logfullqueries")==0) { @@ -2825,13 +2825,13 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme reply.appendf(""); else reply.appendf(""); - unsigned __int64 thash = getTopologyHash(); unsigned __int64 shash; { ReadLockBlock readBlock(packageCrit); shash = allQueryPackages->queryHash(); } - reply.appendf("", shash, thash); + reply.appendf("", + shash, currentTopologyHash, originalTopologyHash); } else if (stricmp(queryName, "control:resetcache")==0) { @@ -2936,13 +2936,13 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme reply.appendf(""); else reply.appendf(""); - unsigned __int64 thash = getTopologyHash(); unsigned __int64 shash; { ReadLockBlock readBlock(packageCrit); shash = allQueryPackages->queryHash(); } - reply.appendf("", shash, thash); + reply.appendf("", + shash, currentTopologyHash, originalTopologyHash); } else if (stricmp(queryName, "control:steppingEnabled")==0) { @@ -3029,7 +3029,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme { topology->setPropBool("@lockDali", false); // Dali will reattach via the timer that checks every so often if can reattach... - saveTopology(); + saveTopology(false); } else if (stricmp(queryName, "control:unsuspend")==0) { @@ -3068,6 +3068,7 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme unknown = true; break; } + currentTopologyHash = getTopologyHash(); if (unknown) throw MakeStringException(ROXIE_UNKNOWN_QUERY, "Unknown query %s", queryName); } @@ -3076,13 +3077,6 @@ class CRoxiePackageSetManager : implements IRoxieQueryPackageManagerSet, impleme { throw MakeStringException(ROXIE_INVALID_INPUT, "Badly formated control query"); } - - hash64_t getTopologyHash() - { - StringBuffer xml; - toXML(topology, xml, 0, XML_SortTags); - return rtlHash64Data(xml.length(), xml.str(), 707018); - } }; extern IRoxieQueryPackageManagerSet *createRoxiePackageSetManager(const IQueryDll *standAloneDll) diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index be4413d4356..084d0e9bff4 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -2441,7 +2441,7 @@ class CLazyKeyIndex : implements IKeyIndex, public CInterface virtual IKeyIndex *queryPart(unsigned idx) { return idx ? NULL : this; } virtual unsigned queryScans() { return realKey ? realKey->queryScans() : 0; } virtual unsigned querySeeks() { return realKey ? realKey->querySeeks() : 0; } - virtual const char *queryFileName() { return keyfile.get(); } + virtual const char *queryFileName() const { return keyfile.get(); } virtual offset_t queryBlobHead() { return checkOpen().queryBlobHead(); } virtual void resetCounts() { if (realKey) realKey->resetCounts(); } virtual offset_t queryLatestGetNodeOffset() const { return realKey ? realKey->queryLatestGetNodeOffset() : 0; } @@ -2659,7 +2659,14 @@ const CJHTreeNode *CNodeCache::getNode(const INodeLoader *keyIndex, unsigned iD, if (!ownedCacheEntry->isReady()) { const CJHTreeNode *node = keyIndex->loadNode(&fetchCycles, pos); - assertex(type == node->getNodeType()); + if (unlikely(type != node->getNodeType())) + { + //This should never happen, but if it does, report as much information as possible to diagnose the issue. + StringBuffer msg; + msg.appendf("Node type mismatch for node %s@%llx (expected %s, got %s)", keyIndex->queryFileName(), pos, cacheTypeText[type], cacheTypeText[node->getNodeType()]); + node->Release(); + throwUnexpectedX(msg); + } //Update the associated size of the entry in the hash table before setting isReady (never evicted until isReady is set) curCache.noteReady(*node); diff --git a/system/jhtree/jhtree.hpp b/system/jhtree/jhtree.hpp index d22531c3b70..8879696ef59 100644 --- a/system/jhtree/jhtree.hpp +++ b/system/jhtree/jhtree.hpp @@ -90,7 +90,7 @@ interface jhtree_decl IKeyIndex : public IKeyIndexBase virtual unsigned querySeeks() = 0; virtual size32_t keyedSize() = 0; virtual bool hasPayload() = 0; - virtual const char *queryFileName() = 0; + virtual const char *queryFileName() const = 0; virtual offset_t queryBlobHead() = 0; virtual void resetCounts() = 0; virtual offset_t queryLatestGetNodeOffset() const = 0; diff --git a/system/jhtree/jhtree.ipp b/system/jhtree/jhtree.ipp index 9280e967f40..93ae8013db7 100644 --- a/system/jhtree/jhtree.ipp +++ b/system/jhtree/jhtree.ipp @@ -78,6 +78,7 @@ interface INodeLoader virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const = 0; virtual const CJHSearchNode *locateFirstLeafNode(IContextLogger *ctx) const = 0; virtual const CJHSearchNode *locateLastLeafNode(IContextLogger *ctx) const = 0; + virtual const char *queryFileName() const = 0; }; class jhtree_decl CKeyIndex : implements IKeyIndex, implements INodeLoader, public CInterface @@ -161,6 +162,8 @@ public: virtual const CJHSearchNode *locateLastLeafNode(IContextLogger *ctx) const override; virtual void mergeStats(CRuntimeStatisticCollection & stats) const override {} + + virtual const char *queryFileName() const = 0; }; class jhtree_decl CMemKeyIndex : public CKeyIndex @@ -170,7 +173,7 @@ private: public: CMemKeyIndex(unsigned _iD, IMemoryMappedFile *_io, const char *_name, bool _isTLK); - virtual const char *queryFileName() { return name.get(); } + virtual const char *queryFileName() const { return name.get(); } virtual const IFileIO *queryFileIO() const override { return nullptr; } // INodeLoader impl. virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const override; @@ -186,7 +189,7 @@ private: public: CDiskKeyIndex(unsigned _iD, IFileIO *_io, const char *_name, bool _isTLK); - virtual const char *queryFileName() { return name.get(); } + virtual const char *queryFileName() const { return name.get(); } virtual const IFileIO *queryFileIO() const override { return io; } // INodeLoader impl. virtual const CJHTreeNode *loadNode(cycle_t * fetchCycles, offset_t offset) const override; diff --git a/system/jlib/jexcept.cpp b/system/jlib/jexcept.cpp index d85aa4fbb82..1cbfb8bb150 100644 --- a/system/jlib/jexcept.cpp +++ b/system/jlib/jexcept.cpp @@ -799,6 +799,24 @@ void throwUnexpectedException(const char * what, const char * function, const ch throw makeStringExceptionV(9999, "Internal Error '%s' in %s() at %s(%d)", what, function, sanitizeSourceFile(file), line); } +void jlib_decl throwUnimplementedException(const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(9999, "UNIMPLEMENTED feature in function %s() at %s(%d)", function, sanitizeSourceFile(file), line); +} + +void jlib_decl throwUnimplementedException(const char * what, const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(-1, "UNIMPLEMENTED feature [%s] in function %s() at %s(%d)", what, function, sanitizeSourceFile(file), line); +} + +void jlib_decl throwUnimplementedException(const char * what, const char *what2, const char * function, const char * file, unsigned line) +{ + printStackReport(); + throw makeStringExceptionV(-1, "UNIMPLEMENTED feature [%s %s] in function %s() at %s(%d)", what, what2, function, sanitizeSourceFile(file), line); +} + void raiseAssertException(const char *assertion, const char *file, unsigned line) { StringBuffer s; diff --git a/system/jlib/jexcept.hpp b/system/jlib/jexcept.hpp index 3e2dfb5b95b..3a80701c673 100644 --- a/system/jlib/jexcept.hpp +++ b/system/jlib/jexcept.hpp @@ -135,7 +135,7 @@ void jlib_decl setTerminateOnSEH(bool set=true); void jlib_decl setProcessAborted(bool _abortVal); __declspec(noreturn) void jlib_decl throwUnexpectedException(const char * function, const char * file, unsigned line) __attribute__((noreturn)); -__declspec(noreturn) void jlib_decl throwUnexpectedException(const char * function, const char * where, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnexpectedException(const char * what, const char * function, const char * file, unsigned line) __attribute__((noreturn)); const char jlib_decl *sanitizeSourceFile(const char *file); @@ -144,9 +144,16 @@ const char jlib_decl *sanitizeSourceFile(const char *file); #define throwUnexpectedX(x) throwUnexpectedException(x, __func__, sanitizeSourceFile(__FILE__), __LINE__) #define assertThrow(x) assertex(x) -#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define UNIMPLEMENTED_X(reason) throw makeStringExceptionV(-1, "UNIMPLEMENTED '" reason "' at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define UNIMPLEMENTED_XY(a,b) throw makeStringExceptionV(-1, "UNIMPLEMENTED " a " %s at %s(%d)", b, sanitizeSourceFile(__FILE__), __LINE__) +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * function, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * what, const char * function, const char * file, unsigned line) __attribute__((noreturn)); +__declspec(noreturn) void jlib_decl throwUnimplementedException(const char * what, const char *what2, const char * function, const char * file, unsigned line) __attribute__((noreturn)); +#define throwUnimplemented() throwUnimplementedException(__func__, sanitizeSourceFile(__FILE__), __LINE__) +#define throwUnimplementedX(x) throwUnimplementedException(x, __func__, sanitizeSourceFile(__FILE__), __LINE__) + +#define UNIMPLEMENTED throwUnimplementedException(__func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_C throwUnimplementedException("CLASSTYPE:", typeid(*this).name(), __func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_X(reason) throwUnimplementedException(reason, __func__, sanitizeSourceFile(__FILE__), __LINE__) +#define UNIMPLEMENTED_XY(a,b) throwUnimplementedException(a, b, __func__, sanitizeSourceFile(__FILE__), __LINE__) IException jlib_decl * deserializeException(MemoryBuffer & in); void jlib_decl serializeException(IException * e, MemoryBuffer & out); diff --git a/system/jlib/jptree.cpp b/system/jlib/jptree.cpp index 28f769af44d..bd8b9acd053 100644 --- a/system/jlib/jptree.cpp +++ b/system/jlib/jptree.cpp @@ -48,8 +48,9 @@ #define WARNLEGACYCOMPARE #define XMLTAG_CONTENT "<>" -#undef UNIMPLEMENTED -#define UNIMPLEMENTED throw MakeIPTException(-1, "UNIMPLEMENTED") +#define UNIMPLEMENTED_IPT throw MakeIPTException(-1, "UNIMPLEMENTED feature in function %s() at %s(%d)", __func__, sanitizeSourceFile(__FILE__), __LINE__) + + #define CHECK_ATTRIBUTE(X) if (X && isAttribute(X)) throw MakeIPTException(PTreeExcpt_XPath_Unsupported, "Attribute usage invalid here"); #define AMBIGUOUS_PATH(X,P) { StringBuffer buf; buf.append(X": ambiguous xpath \"").append(P).append("\""); throw MakeIPTException(PTreeExcpt_XPath_Ambiguity,"%s",buf.str()); } @@ -1914,7 +1915,7 @@ bool PTree::renameProp(const char *xpath, const char *newName) if (strcmp(xpath,"/")==0) // rename of self allowed assuming no parent setName(newName); else if ('[' == *xpath) - UNIMPLEMENTED; + UNIMPLEMENTED_IPT; else if (isAttribute(xpath)) { StringBuffer val; @@ -3506,7 +3507,7 @@ bool PTree::checkPattern(const char *&xxpath) const for (;;) { if (matchElem->isBinary(tProp)) - UNIMPLEMENTED; + UNIMPLEMENTED_IPT; const char *rhs; unsigned rhslength; if (quoteEnd) @@ -6290,7 +6291,7 @@ class CStringBufferMarkupIOAdapter : public CInterfaceOf public: CStringBufferMarkupIOAdapter(StringBuffer &_out) : out(_out) { } virtual void flush() override { } - virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED; return 0; } + virtual size32_t read(size32_t len, void * data) override { UNIMPLEMENTED_IPT; } virtual size32_t write(size32_t len, const void * data) override { out.append(len, (const char *)data); return len; } }; diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 4903eb79f40..e5015e86d20 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -425,10 +425,9 @@ class SecretCacheEntry : public CInterface friend class SecretCache; public: - //A cache entry is initally created that has a create and access time of now, but the checkTimestamp - //is set so that needsRefresh() will return true. + //A cache entry is initally created that has a create and access,and check time of now SecretCacheEntry(cache_timestamp _now, const char * _secretKey) - : secretKey(_secretKey), contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now - 2 * secretTimeoutNs) + : secretKey(_secretKey), contentTimestamp(_now), accessedTimestamp(_now), checkedTimestamp(_now) { } @@ -530,9 +529,10 @@ class SecretCache } //Check to see if a secret exists, and if not add a null entry that has expired. - SecretCacheEntry * getSecret(const std::string & secretKey, cache_timestamp now) + SecretCacheEntry * getSecret(const std::string & secretKey, cache_timestamp now, bool & isNewEntry) { SecretCacheEntry * result; + isNewEntry = false; CriticalBlock block(cs); auto match = secrets.find(secretKey); if (match != secrets.cend()) @@ -542,9 +542,10 @@ class SecretCache } else { - //Insert an entry with a null value that is marked as out of date + //Insert an entry with a null value result = new SecretCacheEntry(now, secretKey.c_str()); secrets.emplace(secretKey, result); + isNewEntry = true; } return result; } @@ -1170,8 +1171,9 @@ static SecretCacheEntry * getSecretEntry(const char * category, const char * nam std::string key(buildSecretKey(category, name, optVaultId, optVersion)); - SecretCacheEntry * match = globalSecretCache.getSecret(key, now); - if (!match->needsRefresh(now)) + bool isNewEntry; + SecretCacheEntry * match = globalSecretCache.getSecret(key, now, isNewEntry); + if (!isNewEntry && !match->needsRefresh(now)) return match; Owned resolved(resolveSecret(category, name, optVaultId, optVersion)); @@ -1314,7 +1316,7 @@ void CSecret::checkUptoDate() const if (secret->needsRefresh(now)) { #ifdef TRACE_SECRETS - DBGLOG("Secret %s is stale updating from %u...", secret->queryTraceName(), secretHash); + DBGLOG("Secret %s is stale updating...", secret->queryTraceName()); #endif //MORE: This could block or fail - in roxie especially it would be better to return the old value try diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index e5ab955c533..a2c0b9b8108 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -25,15 +25,10 @@ #include "opentelemetry/sdk/trace/batch_span_processor_factory.h" #include "opentelemetry/exporters/ostream/span_exporter_factory.h"// auto exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); #include "opentelemetry/exporters/ostream/common_utils.h" -//#define oldForEach ForEach // error: ‘ForEach’ was not declared in this scope -#undef ForEach //opentelemetry defines ForEach #include "opentelemetry/exporters/memory/in_memory_span_exporter_factory.h" #include "opentelemetry/trace/propagation/http_trace_context.h" //opentel_trace::propagation::kTraceParent -#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED #include "opentelemetry/trace/provider.h" //StartSpanOptions #include "opentelemetry/exporters/otlp/otlp_grpc_exporter.h" -#define UNIMPLEMENTED throw makeStringExceptionV(-1, "UNIMPLEMENTED feature at %s(%d)", sanitizeSourceFile(__FILE__), __LINE__) -#define ForEach(i) for((i).first();(i).isValid();(i).next()) #include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h" #include "opentelemetry/exporters/otlp/otlp_http_exporter_factory.h" @@ -43,6 +38,11 @@ #include "opentelemetry/sdk/trace/exporter.h" #include "opentelemetry/sdk/trace/span_data.h" +// NB: undefine after opentelemetry includes, and before HPCC includes where we define. +#undef ForEach //opentelemetry defines ForEach +#undef UNIMPLEMENTED //opentelemetry defines UNIMPLEMENTED + + #include "platform.h" #include "jlib.hpp" #include "jmisc.hpp" diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index b7054529417..041913f351b 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -4252,8 +4252,8 @@ class JLibSecretsTest : public CppUnit::TestFixture CPPUNIT_ASSERT(!secret6->isValid()); CPPUNIT_ASSERT(!secret6->isStale()); - //Sleep so the cache entry should have expired and the value reread since reading ahead - MilliSleep(60); // elapsed=110 = 80 + 30 + //Sleep so the cache entry should have expired (between 80 and 85ms) and the value reread since reading ahead + MilliSleep(60); // elapsed=110 = 50 + 60 CPPUNIT_ASSERT(secret6->isValid()); CPPUNIT_ASSERT(!secret6->isStale()); unsigned version1 = secret6->getVersion(); // Mark the value as accessed, but too early to be refreshed diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 2aa393c5d5c..d5a2c0d57f7 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -1095,7 +1095,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName, SCMStringBuffer eclstr; StringAttr user(workunit.queryUser()); - PROGLOG("Started wuid=%s, user=%s, graph=%s\n", wuid.str(), user.str(), graphName); + PROGLOG("Started wuid=%s, user=%s, graph=%s", wuid.str(), user.str(), graphName); PROGLOG("Query %s loaded", soPath.str()); Owned job = createThorGraph(graphName, workunit, querySo, sendSo, agentEp); diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 719899dc6a9..bd2bcf6ffa0 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -1896,7 +1896,7 @@ class CJobListener : public CSimpleInterface activeJobName.set(wuid); - PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]\n", wuid.get(), user.str(), graphName.get(), maxLogDetail); + PROGLOG("Started wuid=%s, user=%s, graph=%s [log detail level=%u]", wuid.get(), user.str(), graphName.get(), maxLogDetail); PROGLOG("Using query: %s", soPath.str()); if (!getExpertOptBool("slaveDaliClient") && workUnitInfo->getPropBool("Debug/slavedaliclient", false))