diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp index 85cbe50d6b6..c9d803e2061 100644 --- a/common/thorhelper/thorcommon.hpp +++ b/common/thorhelper/thorcommon.hpp @@ -430,7 +430,7 @@ class CStatsContextLogger : public CSimpleInterfaceOf { stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const override + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { stats.merge(from); } diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index bacd694c42f..70a98ae5256 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -19,6 +19,7 @@ #include #include "jlib.hpp" +#include "jconfig.hpp" #include "jcontainerized.hpp" #include "workunit.hpp" #include "jprop.hpp" @@ -14313,17 +14314,36 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP CCycleTimer elapsedTimer; bool multiJobLinger = config.getPropBool("@multiJobLinger", defaultThorMultiJobLinger); + bool thisThor = true; + const char *queue = config.queryProp("@queue"); + const char *tgt = workunit.queryClusterName(); + if (!isEmptyString(tgt)) // don't think should ever happen + { + if (!streq(tgt, queue)) + { + Owned thorTarget = config::getContainerTargets("thor", tgt); + if (!thorTarget->first()) + throw makeStringExceptionV(0, "Thor target not found: %s", tgt); + thisThor = false; + queue = tgt; + } + } // NB: executeGraphOnLingeringThor looks for existing Thor instance that has been used for the same job, // and communicates with it directly if (!multiJobLinger && executeGraphOnLingeringThor(workunit, wfid, graphName)) + { + if (!thisThor) + throw makeStringExceptionV(0, "multiJobLinger mode required to target other thor instances. Target: %s", tgt); PROGLOG("Existing lingering Thor handled graph: %s", graphName); + } else { // If no existing Thor instance, or for a multi linger configuration, // queue the graph, either the thor agent will pick it up and launch a new Thor (up to maxGraphs), // or an existing idle Thor listening on the same queue will pick it up. - VStringBuffer queueName("%s.thor", config.queryProp("@queue")); + + VStringBuffer queueName("%s.thor", queue); DBGLOG("Queueing wuid=%s, graph=%s, on queue=%s, timelimit=%u seconds", wuid.str(), graphName, queueName.str(), timelimit); { diff --git a/common/wuanalysis/anawu.cpp b/common/wuanalysis/anawu.cpp index a8b7aac1c96..79aa20c9495 100644 --- a/common/wuanalysis/anawu.cpp +++ b/common/wuanalysis/anawu.cpp @@ -155,7 +155,7 @@ class WorkunitAnalyserBase public: WorkunitAnalyserBase(); - void analyse(IConstWorkUnit * wu); + void analyse(IConstWorkUnit * wu, const char * optGraph); WuScope * getRootScope() { return LINK(root); } protected: @@ -1282,10 +1282,17 @@ WorkunitAnalyserBase::WorkunitAnalyserBase() : root(new WuScope("", nullptr)) { } -void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu) +void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu, const char * optGraph) { WuScopeFilter filter; filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes); + if (optGraph) + { + //Only include the specified graph, and include everything that matches below that graph + filter.addScopeType(SSTgraph); + filter.addId(optGraph); + filter.setIncludeNesting((unsigned)-1); + } filter.finishedFilter(); collateWorkunitStats(wu, filter); root->connectActivities(); @@ -2079,11 +2086,11 @@ void WorkunitStatsAnalyser::traceDependencies() //--------------------------------------------------------------------------------------------------------------------- -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs) +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs) { WorkunitRuleAnalyser analyser; analyser.applyConfig(options, wu); - analyser.analyse(wu); + analyser.analyse(wu, optGraph); analyser.applyRules(); analyser.update(wu, costPerMs); } @@ -2092,7 +2099,7 @@ void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costRate, { WorkunitRuleAnalyser analyser; analyser.applyConfig(nullptr, wu); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.applyRules(); analyser.print(); if (updatewu) @@ -2114,7 +2121,7 @@ void analyseActivity(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.reportActivity(args); } @@ -2123,7 +2130,7 @@ void analyseDependencies(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.spotCommonPath(args); @@ -2137,7 +2144,7 @@ void analyseOutputDependencyGraph(IConstWorkUnit * wu, IPropertyTree * cfg) { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceDependencies(); @@ -2147,7 +2154,7 @@ void analyseCriticalPath(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceCriticalPaths(args); @@ -2157,7 +2164,7 @@ void analyseHotspots(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); const char * rootScope = nullptr; if (args.ordinality()) @@ -2177,7 +2184,7 @@ void analyseHotspots(WuHotspotResults & results, IConstWorkUnit * wu, IPropertyT { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.findHotspots(cfg->queryProp("@rootScope"), results.totalTime, results.hotspots); results.root.setown(analyser.getRootScope()); diff --git a/common/wuanalysis/anawu.hpp b/common/wuanalysis/anawu.hpp index 8db7d227426..db2f004bb46 100644 --- a/common/wuanalysis/anawu.hpp +++ b/common/wuanalysis/anawu.hpp @@ -26,7 +26,7 @@ #include "anacommon.hpp" -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs); +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs); void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costPerMs, bool updatewu); //--------------------------------------------------------------------------------------------------------------------- diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 4c9732761ff..de2c7c9d229 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -1668,6 +1668,9 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize) unsigned numEmptyXml = 0; ForEachItemIn(idx, sources) { + if (isAborting()) + throwError(DFTERR_CopyAborted); + FilePartInfo & cur = sources.item(idx); StringBuffer s; cur.filename.getPath(s); @@ -1748,8 +1751,7 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize) // Despray from distributed file // Check XMLheader/footer in file level - DistributedFilePropertyLock lock(distributedSource); - IPropertyTree &curProps = lock.queryAttributes(); + IPropertyTree &curProps = distributedSource->queryAttributes(); if (curProps.hasProp(FPheaderLength) && curProps.hasProp(FPfooterLength)) { cur.xmlHeaderLength = curProps.getPropInt(FPheaderLength, 0); diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 405b3641358..0af88caa6e2 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -1590,7 +1590,7 @@ char *EclAgent::getEnv(const char *name, const char *defaultValue) const void EclAgent::selectCluster(const char *newCluster) { - const char *oldCluster = queryWorkUnit()->queryClusterName(); + StringAttr oldCluster = queryWorkUnit()->queryClusterName(); if (getClusterType(clusterType)==HThorCluster) { // If the current cluster is an hthor cluster, it's an error to change it... @@ -1857,6 +1857,27 @@ void EclAgent::setRetcode(int code) retcode = code; } + +void EclAgent::runWorkunitAnalyser(IWorkUnit * w, const char * optGraph) +{ + if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + { + double costPerMs = calculateThorCost(1, getNodes()); + IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); + analyseWorkunit(w, optGraph, analyzerOptions, costPerMs); + } +} + +static constexpr bool defaultAnalyzeWhenComplete = true; +void EclAgent::runWorkunitAnalyserAfterGraph(const char * graph) +{ + if (!wuRead->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) + { + Owned wu(updateWorkUnit()); + runWorkunitAnalyser(wu, graph); + } +} + void EclAgent::doProcess() { #ifdef _DEBUG @@ -2023,15 +2044,22 @@ void EclAgent::doProcess() break; } - if (w->getState() == WUStateCompleted && getClusterType(clusterType)==ThorLCRCluster) + + if (getClusterType(clusterType)==ThorLCRCluster) { - if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + if (w->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) { - double costPerMs = calculateThorCost(1, getNodes()); - IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); - analyseWorkunit(w.get(), analyzerOptions, costPerMs); + switch (w->getState()) + { + case WUStateFailed: + case WUStateAborted: + case WUStateCompleted: + runWorkunitAnalyser(w, nullptr); + break; + } } } + if(w->queryEventScheduledCount() > 0) switch(w->getState()) { diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp index 53ece5514ed..5362b60c929 100644 --- a/ecl/eclagent/eclagent.ipp +++ b/ecl/eclagent/eclagent.ipp @@ -431,6 +431,8 @@ private: EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName); virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq); virtual IDebuggableContext *queryDebugContext() const { return debugContext; }; + void runWorkunitAnalyser(IWorkUnit * w, const char * optGraph); + void runWorkunitAnalyserAfterGraph(const char * optGraph); //protected by critical section EclGraph * addGraph(const char * graphName); diff --git a/ecl/eclagent/eclgraph.cpp b/ecl/eclagent/eclgraph.cpp index 26e4ca71bc4..a14266c4c07 100644 --- a/ecl/eclagent/eclgraph.cpp +++ b/ecl/eclagent/eclgraph.cpp @@ -1603,7 +1603,16 @@ void EclAgent::executeGraph(const char * graphName, bool realThor, size32_t pare { if (isStandAloneExe) throw MakeStringException(0, "Cannot execute Thor Graph in standalone mode"); - executeThorGraph(graphName, *wuRead, *agentTopology); + try + { + executeThorGraph(graphName, *wuRead, *agentTopology); + runWorkunitAnalyserAfterGraph(graphName); + } + catch (...) + { + runWorkunitAnalyserAfterGraph(graphName); + throw; + } } else { diff --git a/ecl/regress/issue31961.ecl b/ecl/regress/issue31961.ecl new file mode 100644 index 00000000000..7feca48cdd3 --- /dev/null +++ b/ecl/regress/issue31961.ecl @@ -0,0 +1,44 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + + + +strRec := { string x; }; + +myRec := RECORD + STRING a; + STRING b; + STRING c; + STRING d; + DATASET(strRec) e; +END; + + +makeDs(unsigned start) := DATASET(100, TRANSFORM(strRec, SELF.x := (STRING)(start + counter))); + +myRec t(unsigned cnt) := TRANSFORM + SELF.a := (STRING)cnt; + SELF.b := (STRING)HASH32(cnt); + SELF.c := (STRING)HASH64(cnt); + SELF.d := (STRING)HASH64(cnt+1); + SELF.e := makeDs(cnt); +END; + +i := DATASET(1000000, t(COUNTER)); +s := SORT(i, a, HINT(sortCompBlkSz(1000)), LOCAL); +c := COUNT(NOFOLD(s)); +output(c); diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 84c45e73947..bc4b85a6248 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -15,20 +15,20 @@ "@fluentui/react-hooks": "8.7.0", "@fluentui/react-icons-mdl2": "1.3.59", "@fluentui/react-migration-v8-v9": "9.6.3", - "@hpcc-js/chart": "2.83.2", - "@hpcc-js/codemirror": "2.61.3", - "@hpcc-js/common": "2.71.16", - "@hpcc-js/comms": "2.92.0", + "@hpcc-js/chart": "2.83.3", + "@hpcc-js/codemirror": "2.61.4", + "@hpcc-js/common": "2.71.17", + "@hpcc-js/comms": "2.92.1", "@hpcc-js/dataflow": "8.1.6", - "@hpcc-js/eclwatch": "2.74.2", - "@hpcc-js/graph": "2.85.14", - "@hpcc-js/html": "2.42.19", - "@hpcc-js/layout": "2.49.21", - "@hpcc-js/map": "2.77.20", - "@hpcc-js/other": "2.15.21", - "@hpcc-js/phosphor": "2.18.7", - "@hpcc-js/react": "2.53.15", - "@hpcc-js/tree": "2.40.16", + "@hpcc-js/eclwatch": "2.74.3", + "@hpcc-js/graph": "2.85.15", + "@hpcc-js/html": "2.42.20", + "@hpcc-js/layout": "2.49.22", + "@hpcc-js/map": "2.77.21", + "@hpcc-js/other": "2.15.22", + "@hpcc-js/phosphor": "2.18.8", + "@hpcc-js/react": "2.53.16", + "@hpcc-js/tree": "2.40.17", "@hpcc-js/util": "2.51.0", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", @@ -1806,35 +1806,35 @@ } }, "node_modules/@hpcc-js/api": { - "version": "2.12.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.16.tgz", - "integrity": "sha512-lmRvwoAHWcrTSKEbe/SR0Y61p4j/+VRnu7CViBe5wQ2nCrK9yFTLZzDlmHhvMIDyVJj4khI4/4ZZCUI13zfuzA==", + "version": "2.12.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.17.tgz", + "integrity": "sha512-NuqjPdxnfbpFQ0e2c0ZBC/hYItPDOTiysO+xfR03SSTqZqxdcME1EEYHI6/wIAb3B+rQ3gk3Xss7EvKq6t+puw==", "dependencies": { - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/chart": { - "version": "2.83.2", - "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.83.2.tgz", - "integrity": "sha512-PePaV/68if4dp+iBDpmBwRbTfOsKht8DkZ7B8NafKfGxL1+khCXf4p4Q/Dlgqig1OfVt35z1eR4lbEUrARhVJw==", + "version": "2.83.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.83.3.tgz", + "integrity": "sha512-CKTnjQ4PhK/g7o/ZvNA3o+AkCLIu/ttkkqDyPmiFQU9an7qXfAVyJyPk9e2/QCeRaTQV1YmMUAj94iZvK9E0Gg==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/codemirror": { - "version": "2.61.3", - "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.61.3.tgz", - "integrity": "sha512-GKLuro8GiMUKu9sCoIsyIXMRbTogG3xGb/yvAuoAKcJQRyJsVamLB7ohyjZkcgY7/JWa3+UBUcABOxiWqG8M/Q==", + "version": "2.61.4", + "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.61.4.tgz", + "integrity": "sha512-rscy1L5EcRhRtldjjwdurxC8RLWW8KY+B8EYj/XXH25blpvlt3P05Bdd6kotBIG18sV33sezaydhM7dqs+iltg==", "dependencies": { - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/common": { - "version": "2.71.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.16.tgz", - "integrity": "sha512-hz5i9zUXBJrXW5tl30XwgFXwJ2nipzLD9pXQrg1Rw8zfXkQ1Xax22RvGZdASAGPsmHxefyWTK7fpcJd+ipOGOg==", + "version": "2.71.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.17.tgz", + "integrity": "sha512-Fyo7U/1hFgV7ZEkiOSj92UYEEtj0T73KV7w09yyD36paSS0wFLTNaFoiFJRVRgmQF8P/LVBgC1lrB7LnYdfFNA==", "dependencies": { "@hpcc-js/util": "^2.51.0", "@types/d3-array": "1.2.12", @@ -1855,9 +1855,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.92.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.92.0.tgz", - "integrity": "sha512-hGWFUIywb/DHR/yk43C911JY9mwNrion/wt71adfjbckjQJ267GjPkw4tyz8K1YGt5NgCCW+njnR6lAkpjYGfw==", + "version": "2.92.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.92.1.tgz", + "integrity": "sha512-nx4JJUSpU1m/Yd53PsbEL26DQFx+3UF0Pelk5O0BY2eFGpPVaQ9jHeWvuMJCHHpHcbxUtWN5nlwZlZgE9CFptA==", "dependencies": { "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/util": "^2.51.0", @@ -1906,11 +1906,11 @@ } }, "node_modules/@hpcc-js/dgrid": { - "version": "2.32.19", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.19.tgz", - "integrity": "sha512-nFKWjepBJIceN2sTMk8N283OFvU5zwfFAeGqBnT3iQRO2vQRaJzZt4G+9xtgVPbnyWuGiqHhIxYoGJLUOMpLbQ==", + "version": "2.32.20", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.20.tgz", + "integrity": "sha512-hCD6nIfWT1RWHBTIAqMs2uJUQSE7hBKLqa6pLFRmlByL7Egev+m5ErR4+aMn9NjqhjE0/HqpwwYnTNoMoVw83w==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/dgrid-shim": "^2.24.10", "@hpcc-js/util": "^2.51.0" @@ -1922,63 +1922,63 @@ "integrity": "sha512-4PD4GvKn2/HQvgzeP+Gd0Halj4KySk0QW1C7dqfyNWV8AUaseT9SSUvyu2ftGPUrzq65sJ0fSaq4zh3Js9dbaQ==" }, "node_modules/@hpcc-js/dgrid2": { - "version": "2.3.18", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.18.tgz", - "integrity": "sha512-7OtEREk9xJYfjDGGP9abSvQWQCDTgwYIx+OfXqFArb031FhTC1rWrkc5svySRb/0VVVAvsQFkHK435GaNy6PHQ==", + "version": "2.3.19", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.19.tgz", + "integrity": "sha512-K0GCX2GR+sayN1glQAEmqdFt+ZcvF3ZU3xwO/LpcA6HGqnN3ena+CeCEE1ufqfLICwPu5P/X+kMd+GJoDY51+w==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/eclwatch": { - "version": "2.74.2", - "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.74.2.tgz", - "integrity": "sha512-FY5CQ/Pezq5enRZtVXzmxV2utv+Fiq7Gn7guMz2IhYWmenNDgclIrfKHeXL8nISJsPNl/VJOHCwyxBWuhuGBdw==", - "dependencies": { - "@hpcc-js/codemirror": "^2.61.3", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/comms": "^2.92.0", - "@hpcc-js/dgrid": "^2.32.19", - "@hpcc-js/graph": "^2.85.14", - "@hpcc-js/layout": "^2.49.21", - "@hpcc-js/phosphor": "^2.18.7", - "@hpcc-js/timeline": "^2.51.24", - "@hpcc-js/tree": "^2.40.16", + "version": "2.74.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.74.3.tgz", + "integrity": "sha512-tsJfXAbREXNXAzui8Mc7Vb9J2xmc1A40I2+pTTOFnVeHPv8bzDvc5sGQXgRrkqqOkeMwzGsnlpbVmC7zTZ33UA==", + "dependencies": { + "@hpcc-js/codemirror": "^2.61.4", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/comms": "^2.92.1", + "@hpcc-js/dgrid": "^2.32.20", + "@hpcc-js/graph": "^2.85.15", + "@hpcc-js/layout": "^2.49.22", + "@hpcc-js/phosphor": "^2.18.8", + "@hpcc-js/timeline": "^2.51.25", + "@hpcc-js/tree": "^2.40.17", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/graph": { - "version": "2.85.14", - "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.14.tgz", - "integrity": "sha512-grofTqK944A8b/LgigDJHBuM9R9+JIDYfqA5wBssbvty3MtLAuN2seoGFn+I7UEojrCggQllKSxNyi/OXoJrCQ==", - "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/html": "^2.42.19", - "@hpcc-js/react": "^2.53.15", + "version": "2.85.15", + "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.15.tgz", + "integrity": "sha512-1LGhS4tywbCPs6b0XObLRiuf3fU16QysTrIA/7F6FUB7w5ay6oiR8tzGTP87SNHZ7fvZ36TAXOyKi/jHJbaf1A==", + "dependencies": { + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/html": "^2.42.20", + "@hpcc-js/react": "^2.53.16", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/html": { - "version": "2.42.19", - "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.19.tgz", - "integrity": "sha512-qocVJXQvwUVaHVXQvn8gIZCXfNHiQOVuMai5wyegYH9KgWJUX3MjUNGHCEYpMsBkk6LBX+D+3myC+VFGlLSgjg==", + "version": "2.42.20", + "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.20.tgz", + "integrity": "sha512-LozHVD0THMJ1IUjbTsmzskoWYobc5siv1S4rgl6sAy1R8etnTgWkpDMgmIFLNS97A3XizpCebJwBfeEN6KsDpg==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/layout": { - "version": "2.49.21", - "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.21.tgz", - "integrity": "sha512-gZSqCBrLDriWW9mhw1bqUL/dzNCsf372CnPjsMLEuMdln7TOOTQm5L0BCjMPVf5kMgyZeT2xRZGr7YURPJIw+g==", + "version": "2.49.22", + "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.22.tgz", + "integrity": "sha512-iNCUUpsA3y6bRQC087Enix3d5vpXoQKxr8FoYVkJFji/UooWU2zJelRPwx7Ky8e1t6Jz2uYxqWI30J5wy8h0HA==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/chart": "^2.83.2", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/dgrid2": "^2.3.18" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/chart": "^2.83.3", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/dgrid2": "^2.3.19" } }, "node_modules/@hpcc-js/leaflet-shim": { @@ -1991,36 +1991,36 @@ } }, "node_modules/@hpcc-js/map": { - "version": "2.77.20", - "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.20.tgz", - "integrity": "sha512-smA6i2viO/DsEaNGIbRKLxHTLWEO8qd7nBgtEFOYugyiiIeyZaBtHVEeTEpOMJAv672L+SxbxAteSbfbdTdd/w==", - "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/graph": "^2.85.14", - "@hpcc-js/layout": "^2.49.21", + "version": "2.77.21", + "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.21.tgz", + "integrity": "sha512-LJHDvpvpllYlapQ9xzqw46oMCyn3WfLNIEeBadrtIoxr7xLFrS2dEXkgydv2BwbRIZNHgjXZPqKfbsWBHqph5w==", + "dependencies": { + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/graph": "^2.85.15", + "@hpcc-js/layout": "^2.49.22", "@hpcc-js/leaflet-shim": "^2.3.5", - "@hpcc-js/other": "^2.15.21", + "@hpcc-js/other": "^2.15.22", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/other": { - "version": "2.15.21", - "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.21.tgz", - "integrity": "sha512-QUIlQv7nP9+fKNdE8458pqy/cYrEZnVYXBne8uSu2h5q64VSSreCZmO7XY/Rjxf822RigYlZ+fC8K2fZiDOULw==", + "version": "2.15.22", + "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.22.tgz", + "integrity": "sha512-r1dv7Fswrf9SBbmr437k+CpEvzZJJjpGSUCrF2JrMztHpsFZed4qlo9agIVp5RksXxQf7SPMI9IXWthYIUG5LQ==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/layout": "^2.49.21" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/layout": "^2.49.22" } }, "node_modules/@hpcc-js/phosphor": { - "version": "2.18.7", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.18.7.tgz", - "integrity": "sha512-iSQX6vIpawQPbDVhc/CbH8Z4ysSzb+uFjeasd1zIfE77Km5ImH9IiI9OZMOoFYi1zCTCfce/Y7NLC8ADOgg2XQ==", + "version": "2.18.8", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.18.8.tgz", + "integrity": "sha512-/D7lXuuPoeUuCmaBv/JMjmThKu3zl0j2/m7LxTJ9ltFmu7KkMNQK6EYLc0+VRVWrxa9P8hXMlLCXN0YlXtKTww==", "dependencies": { - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/other": "^2.15.21", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/other": "^2.15.22", "@hpcc-js/phosphor-shim": "^2.14.6", "@hpcc-js/util": "^2.51.0" } @@ -2045,34 +2045,34 @@ } }, "node_modules/@hpcc-js/react": { - "version": "2.53.15", - "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.15.tgz", - "integrity": "sha512-X8e1lIk4oRXFNTFxrcZ1YbJDi6t7IU431Lzq0nTIFJqWIDRZgjJ3gKxSGcjtjNYzhUqq4A9KfcdvKNE+wHrdjw==", + "version": "2.53.16", + "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.16.tgz", + "integrity": "sha512-pJ0/hE2MOCnaWFWBRyx1TEXV6x0bwPkoEKmwmGb72pCJQoVPaSKQ8bEi+UOHX+xxw0TF5x8u7fJHUjxvdcYMmQ==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10" } }, "node_modules/@hpcc-js/timeline": { - "version": "2.51.24", - "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.24.tgz", - "integrity": "sha512-QNgXhJ6/hQHfP2Lge2zL1X5ERI813KKpFN+DNFqufhWoZIT/7x3kr1If8r1mC74hYt4xqkFAdoveEepFT+lYhQ==", + "version": "2.51.25", + "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.25.tgz", + "integrity": "sha512-SS/67TomcrJaaMrtWYs9fk/TK0hr/ve/oBVux5Ibnph826xBB3Z5wt3tzfWYgzlvcsdGPFi6TF4FKFtYOTX9LA==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/chart": "^2.83.2", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/html": "^2.42.19", - "@hpcc-js/layout": "^2.49.21", - "@hpcc-js/react": "^2.53.15" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/chart": "^2.83.3", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/html": "^2.42.20", + "@hpcc-js/layout": "^2.49.22", + "@hpcc-js/react": "^2.53.16" } }, "node_modules/@hpcc-js/tree": { - "version": "2.40.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.16.tgz", - "integrity": "sha512-UCFA3ky9aB0XqrN4PyNmwkY3zl3VSc4araEfHpjtOcT7r7pUVJNEG+KjYPkCTUvvKYoPIuE2FBGtr6ec0bM5Aw==", + "version": "2.40.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.17.tgz", + "integrity": "sha512-Z8uTo6281tcTMLUmNynYJEn5MiS5qkm76FcalgMJKaOduoUXL1YezDGFqo23y61JL6GpHdVRz/go9B5uI+Sapg==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/util": { diff --git a/esp/src/package.json b/esp/src/package.json index a7685259e97..f823572534d 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -41,20 +41,20 @@ "@fluentui/react-hooks": "8.7.0", "@fluentui/react-icons-mdl2": "1.3.59", "@fluentui/react-migration-v8-v9": "9.6.3", - "@hpcc-js/chart": "2.83.2", - "@hpcc-js/codemirror": "2.61.3", - "@hpcc-js/common": "2.71.16", - "@hpcc-js/comms": "2.92.0", + "@hpcc-js/chart": "2.83.3", + "@hpcc-js/codemirror": "2.61.4", + "@hpcc-js/common": "2.71.17", + "@hpcc-js/comms": "2.92.1", "@hpcc-js/dataflow": "8.1.6", - "@hpcc-js/eclwatch": "2.74.2", - "@hpcc-js/graph": "2.85.14", - "@hpcc-js/html": "2.42.19", - "@hpcc-js/layout": "2.49.21", - "@hpcc-js/map": "2.77.20", - "@hpcc-js/other": "2.15.21", - "@hpcc-js/phosphor": "2.18.7", - "@hpcc-js/react": "2.53.15", - "@hpcc-js/tree": "2.40.16", + "@hpcc-js/eclwatch": "2.74.3", + "@hpcc-js/graph": "2.85.15", + "@hpcc-js/html": "2.42.20", + "@hpcc-js/layout": "2.49.22", + "@hpcc-js/map": "2.77.21", + "@hpcc-js/other": "2.15.22", + "@hpcc-js/phosphor": "2.18.8", + "@hpcc-js/react": "2.53.16", + "@hpcc-js/tree": "2.40.17", "@hpcc-js/util": "2.51.0", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", diff --git a/esp/src/src-react/components/DFUWorkunits.tsx b/esp/src/src-react/components/DFUWorkunits.tsx index e0032f2a383..39170270844 100644 --- a/esp/src/src-react/components/DFUWorkunits.tsx +++ b/esp/src/src-react/components/DFUWorkunits.tsx @@ -133,7 +133,7 @@ export const DFUWorkunits: React.FunctionComponent = ({ JobName: { label: nlsHPCC.JobName, width: 220 }, ClusterName: { label: nlsHPCC.Cluster, width: 70 }, StateMessage: { label: nlsHPCC.State, width: 70 }, - PCTDone: { + PercentDone: { label: nlsHPCC.PctComplete, width: 80, sortable: true, }, TimeStarted: { label: nlsHPCC.TimeStarted, width: 100, sortable: true }, diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 7f43a368726..8d63d3ae8f5 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -17,12 +17,12 @@ const logger = scopedLogger("src-react/components/ECLArchive.tsx"); const scopeFilterDefault: Partial = { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }; const nestedFilterDefault: WsWorkunits.NestedFilter = { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }; interface ECLArchiveProps { diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 01a1e53eca8..14939bd6164 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -33,7 +33,7 @@ const FilterFields: Fields = { "SuperFiles": { type: "checkbox", label: nlsHPCC.SuperFiles }, "Indexes": { type: "checkbox", label: nlsHPCC.Indexes }, "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles, disabled: (params: Fields) => !!params?.SuperFiles?.value || !!params?.LogicalFiles?.value }, - "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster }, + "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster, multiSelect: true, valueSeparator: "," }, "FileSizeFrom": { type: "string", label: nlsHPCC.FromSizes, placeholder: "4096" }, "FileSizeTo": { type: "string", label: nlsHPCC.ToSizes, placeholder: "16777216" }, "FileType": { type: "file-type", label: nlsHPCC.FileType }, diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index a4c43170dac..5373bb6ab59 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -198,17 +198,17 @@ export const Metrics: React.FunctionComponent = ({ .request({ ScopeFilter: { MaxDepth: 3, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: ["WhenStarted", "TimeElapsed", "TimeLocalExecute"] } + Properties: ["WhenStarted", "TimeElapsed", "TimeLocalExecute"] }, ScopeOptions: { IncludeId: true, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index a00a73e4670..b2047850ee7 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -1,5 +1,6 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Icon, Image, Link } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, DetailsRow, ICommandBarItemProps, IDetailsRowProps, Icon, Image, Link } from "@fluentui/react"; +import { hsl as d3Hsl } from "@hpcc-js/common"; import { SizeMe } from "react-sizeme"; import { CreateWUQueryStore, defaultSort, emptyFilter, Get, WUQueryStore, formatQuery } from "src/ESPWorkunit"; import * as WsWorkunits from "src/WsWorkunits"; @@ -7,6 +8,8 @@ import { formatCost } from "src/Session"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; +import { useUserStore } from "../hooks/store"; +import { useLogicalClustersPalette } from "../hooks/platform"; import { calcSearch, pushParams } from "../util/history"; import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; @@ -61,12 +64,14 @@ export const Workunits: React.FunctionComponent = ({ const [showFilter, setShowFilter] = React.useState(false); const { currentUser } = useMyAccount(); const [uiState, setUIState] = React.useState({ ...defaultUIState }); + const [showTimeline, setShowTimeline] = useUserStore("workunits_showTimeline", true); const { selection, setSelection, pageNum, setPageNum, pageSize, setPageSize, total, setTotal, refreshTable } = useFluentStoreState({ page }); + const [, , palette] = useLogicalClustersPalette(); // Refresh on focus --- const isMounted = useIsMounted(); @@ -243,7 +248,15 @@ export const Workunits: React.FunctionComponent = ({ pushParams(filter); } }, - ], [currentUser, filter, hasFilter, refreshTable, selection, setShowAbortConfirm, setShowDeleteConfirm, store, total, uiState.hasNotCompleted, uiState.hasNotProtected, uiState.hasProtected, uiState.hasSelection]); + { key: "divider_5", itemType: ContextualMenuItemType.Divider, onRender: () => }, + { + key: "timeline", text: nlsHPCC.Timeline, canCheck: true, checked: showTimeline, iconProps: { iconName: "TimelineProgress" }, + onClick: () => { + setShowTimeline(!showTimeline); + refreshTable.call(); + } + }, + ], [currentUser.username, filter, hasFilter, refreshTable, selection, setShowAbortConfirm, setShowDeleteConfirm, setShowTimeline, showTimeline, store, total, uiState.hasNotCompleted, uiState.hasNotProtected, uiState.hasProtected, uiState.hasSelection]); // Selection --- React.useEffect(() => { @@ -274,6 +287,34 @@ export const Workunits: React.FunctionComponent = ({ setUIState(state); }, [selection]); + const renderRowTimings = React.useCallback((props: IDetailsRowProps, size: { readonly width: number; readonly height: number; }) => { + if (showTimeline && props) { + const total = props.item.timings.page.end - props.item.timings.page.start; + const startPct = 100 - (props.item.timings.start - props.item.timings.page.start) / total * 100; + const endPct = 100 - (props.item.timings.end - props.item.timings.page.start) / total * 100; + const backgroundColor = palette(props.item.Cluster); + const borderColor = d3Hsl(backgroundColor).darker().toString(); + + return
+ +
+
; + } + return ; + }, [palette, showTimeline]); + return } main={ @@ -293,6 +334,7 @@ export const Workunits: React.FunctionComponent = ({ setSelection={setSelection} setTotal={setTotal} refresh={refreshTable} + onRenderRow={showTimeline ? props => renderRowTimings(props, size) : undefined} >
@@ -309,7 +351,7 @@ export const Workunits: React.FunctionComponent = ({ setPageNum={setPageNum} setPageSize={setPageSize} total={total} - >} + >} footerStyles={{}} />; }; diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 08ccb145962..fff30e920ca 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IDetailsList } from "@fluentui/react"; +import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IDetailsList, IRenderFunction, IDetailsRowProps } from "@fluentui/react"; import { Pagination } from "@fluentui/react-experiments/lib/Pagination"; import { useConst, useId, useMount, useOnEvent } from "@fluentui/react-hooks"; import { BaseStore, Memory, QueryRequest, QuerySortItem } from "src/store/Memory"; @@ -193,6 +193,7 @@ interface FluentStoreGridProps { refresh: RefreshTable, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, + onRenderRow?: IRenderFunction } const FluentStoreGrid: React.FunctionComponent = ({ @@ -206,6 +207,7 @@ const FluentStoreGrid: React.FunctionComponent = ({ refresh, setSelection, setTotal, + onRenderRow }) => { const memoizedColumns = useDeepMemo(() => columns, [], [columns]); const [sorted, setSorted] = React.useState(sort); @@ -320,6 +322,7 @@ const FluentStoreGrid: React.FunctionComponent = ({ onColumnHeaderClick={onColumnClick} onRenderDetailsHeader={renderDetailsHeader} onColumnResize={columnResize} + onRenderRow={onRenderRow} styles={gridStyles(height)} /> ; @@ -334,7 +337,8 @@ interface FluentGridProps { height?: string, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, - refresh: RefreshTable + refresh: RefreshTable, + onRenderRow?: IRenderFunction } export const FluentGrid: React.FunctionComponent = ({ @@ -346,7 +350,8 @@ export const FluentGrid: React.FunctionComponent = ({ height, setSelection, setTotal, - refresh + refresh, + onRenderRow }) => { const constStore = useConst(() => new Memory(primaryID, alphaNumColumns)); @@ -357,7 +362,7 @@ export const FluentGrid: React.FunctionComponent = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [constStore, data, /*refresh*/]); - return + return ; }; @@ -372,7 +377,8 @@ interface FluentPagedGridProps { height?: string, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, - refresh: RefreshTable + refresh: RefreshTable, + onRenderRow?: IRenderFunction } export const FluentPagedGrid: React.FunctionComponent = ({ @@ -386,7 +392,8 @@ export const FluentPagedGrid: React.FunctionComponent = ({ height, setSelection, setTotal, - refresh + refresh, + onRenderRow }) => { const [page, setPage] = React.useState(pageNum - 1); const [sortBy, setSortBy] = React.useState(sort); @@ -407,7 +414,7 @@ export const FluentPagedGrid: React.FunctionComponent = ({ setPage(_page); }, [pageNum]); - return + return ; }; diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index ebfd6e2247c..f2c6b66a25d 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -82,6 +82,7 @@ interface AsyncDropdownProps { required?: boolean; disabled?: boolean; multiSelect?: boolean; + valueSeparator?: string; errorMessage?: string; onChange?: (event: React.FormEvent, option?: IDropdownOption | IDropdownOption[], index?: number) => void; placeholder?: string; @@ -95,6 +96,7 @@ const AsyncDropdown: React.FunctionComponent = ({ required = false, disabled, multiSelect = false, + valueSeparator = "|", errorMessage, onChange, placeholder, @@ -114,7 +116,7 @@ const AsyncDropdown: React.FunctionComponent = ({ const [selectedItems, setSelectedItems] = React.useState([]); const changeSelectedItems = React.useCallback(() => { - const keys = selectedKey !== "" ? selectedKey.split("|") : []; + const keys = selectedKey !== "" ? selectedKey.split(valueSeparator) : []; let items = [...selectedItems]; if (keys.length === items.length) return; if (selectedKeys !== "" && selOptions.length && selectedKey === "") { @@ -123,16 +125,16 @@ const AsyncDropdown: React.FunctionComponent = ({ } items = keys.map(key => { return { key: key, text: key }; }); if (!items.length) return; - if (items.map(item => item.key).join("|") === selectedKey) { + if (items.map(item => item.key).join(valueSeparator) === selectedKey) { // do nothing, unless if (!selectedItems.length) { setSelectedItems(items); } } else { - setSelectedKeys(items.map(item => item.key).join("|")); + setSelectedKeys(items.map(item => item.key).join(valueSeparator)); setSelectedItems(items); } - }, [selectedKey, selectedKeys, selectedItems, selOptions]); + }, [selectedKey, selectedKeys, selectedItems, selOptions, valueSeparator]); React.useEffect(() => { // only on mount, pre-populate selectedItems from url @@ -170,7 +172,7 @@ const AsyncDropdown: React.FunctionComponent = ({ React.useEffect(() => { if (multiSelect) { if (!selectedItems.length && selectedKey === "") return; - if (selectedItems.map(item => item.key).join("|") === selectedKey) return; + if (selectedItems.map(item => item.key).join(valueSeparator) === selectedKey) return; onChange(undefined, selectedItems, null); } else { if (!selectedItem || selectedItem?.key === selectedKey) return; @@ -178,7 +180,7 @@ const AsyncDropdown: React.FunctionComponent = ({ onChange(undefined, selectedItem, selectedIdx); } } - }, [onChange, multiSelect, selectedItem, selectedIdx, selectedKey, selectedItems]); + }, [onChange, multiSelect, selectedItem, selectedIdx, selectedKey, selectedItems, valueSeparator]); if (multiSelect) { return options === undefined ? @@ -348,11 +350,14 @@ interface QueriesActiveStateField extends BaseField { interface TargetClusterField extends BaseField { type: "target-cluster"; multiSelect?: boolean; + valueSeparator?: string; value?: string; } interface TargetGroupField extends BaseField { type: "target-group"; + multiSelect?: boolean; + valueSeparator?: string; value?: string; } @@ -1160,16 +1165,18 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a break; case "target-cluster": field.value = field.value !== undefined ? field.value : ""; + field.valueSeparator = field.valueSeparator !== undefined ? field.valueSeparator : "|"; retVal.push({ id: fieldID, label: field.label, field: { if (field.multiSelect) { - onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join("|")); + onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join(field.valueSeparator)); } else { onChange(fieldID, (row as IDropdownOption).key); } @@ -1210,6 +1217,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a break; case "target-group": field.value = field.value !== undefined ? field.value : ""; + field.valueSeparator = field.valueSeparator !== undefined ? field.valueSeparator : ","; retVal.push({ id: fieldID, label: field.label, @@ -1217,7 +1225,15 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a key={fieldID} required={field.required} selectedKey={field.value} - onChange={(ev, row: IDropdownOption) => onChange(fieldID, row.key)} + multiSelect={field.multiSelect} + valueSeparator={field.valueSeparator} + onChange={(ev, row) => { + if (field.multiSelect) { + onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join(field.valueSeparator)); + } else { + onChange(fieldID, (row as IDropdownOption).key); + } + }} placeholder={field.placeholder} /> }); diff --git a/esp/src/src-react/hooks/metrics.ts b/esp/src/src-react/hooks/metrics.ts index efdfa3da2ea..d377997ac92 100644 --- a/esp/src/src-react/hooks/metrics.ts +++ b/esp/src/src-react/hooks/metrics.ts @@ -104,12 +104,12 @@ export enum FetchStatus { const scopeFilterDefault: Partial = { MaxDepth: 999999, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }; const nestedFilterDefault: WsWorkunits.NestedFilter = { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }; export function useWorkunitMetrics( diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index 6c634aed36e..8e90b4ab363 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -6,6 +6,7 @@ import { Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms"; import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session"; import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo"; import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; +import { Palette } from "@hpcc-js/common"; const logger = scopedLogger("src-react/hooks/platform.ts"); @@ -36,14 +37,18 @@ export function useBuildInfo(): [BuildInfo, { isContainer: boolean, currencyCode return [buildInfo, { isContainer, currencyCode, opsCategory }]; } +let g_targetCluster: Promise; export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined, WsTopology.TpLogicalCluster | undefined] { const [targetClusters, setTargetClusters] = React.useState(); const [defaultCluster, setDefaultCluster] = React.useState(); React.useEffect(() => { - const topology = Topology.attach({ baseUrl: "" }); + if (!g_targetCluster) { + const topology = Topology.attach({ baseUrl: "" }); + g_targetCluster = topology.fetchLogicalClusters(); + } let active = true; - topology.fetchLogicalClusters().then(response => { + g_targetCluster.then(response => { if (active) { setTargetClusters(response); let firstRow: WsTopology.TpLogicalCluster; @@ -70,6 +75,22 @@ export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined return [targetClusters, defaultCluster]; } +export function useLogicalClustersPalette(): [WsTopology.TpLogicalCluster[] | undefined, WsTopology.TpLogicalCluster | undefined, Palette.OrdinalPaletteFunc] { + const [targetClusters, defaultCluster] = useLogicalClusters(); + + const palette = useConst(() => Palette.ordinal("workunits", ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"])); + + React.useEffect(() => { + if (targetClusters) { + targetClusters.forEach(cluster => { + palette(cluster.Name); + }); + } + }, [palette, targetClusters]); + + return [targetClusters, defaultCluster, palette]; +} + let wuCheckFeaturesPromise; export const fetchCheckFeatures = () => { if (!wuCheckFeaturesPromise) { diff --git a/esp/src/src/ECLArchiveWidget.ts b/esp/src/src/ECLArchiveWidget.ts index 2f5efaa752a..8076a41433e 100644 --- a/esp/src/src/ECLArchiveWidget.ts +++ b/esp/src/src/ECLArchiveWidget.ts @@ -171,7 +171,7 @@ export class ECLArchiveWidget { const scopesOptions: RecursivePartial = { ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }, ScopeOptions: { IncludeMatchedScopesInResults: true, @@ -189,7 +189,7 @@ export class ECLArchiveWidget { }, NestedFilter: { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }, PropertiesToReturn: { AllStatistics: true, diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 428eb156107..77b406e53fd 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -917,7 +917,7 @@ const Workunit = declare([ESPUtil.Singleton], { // jshint ignore:line return (this._hpccWU as HPCCWorkunit).fetchDetails({ ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }, ScopeOptions: { IncludeMatchedScopesInResults: true, @@ -935,7 +935,7 @@ const Workunit = declare([ESPUtil.Singleton], { // jshint ignore:line }, NestedFilter: { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }, PropertiesToReturn: { AllStatistics: false, @@ -1085,8 +1085,50 @@ export function CreateWUQueryStore(): BaseStore { + const page = { + start: undefined, + end: undefined + }; + const data = response.Workunits.ECLWorkunit.map(wu => { + const start = Utility.wuidToDateTime(wu.Wuid); + if (!page.start || page.start > start) { + page.start = start; + } + let timePartsSection = 0; + const end = new Date(start); + const timeParts = wu.TotalClusterTime.split(":"); + while (timeParts.length) { + const timePart = timeParts.pop(); + switch (timePartsSection) { + case 0: + end.setSeconds(end.getSeconds() + +timePart); + break; + case 1: + end.setMinutes(end.getMinutes() + +timePart); + break; + case 2: + end.setHours(end.getHours() + +timePart); + break; + case 3: + end.setDate(end.getDate() + +timePart); + break; + } + ++timePartsSection; + } + if (!page.end || page.end < end) { + page.end = end; + } + return { + ...Get(wu.Wuid, wu), + timings: { + start, + end, + page + } + }; + }); return { - data: response.Workunits.ECLWorkunit.map(wu => Get(wu.Wuid, wu)), + data, total: response.NumWUs }; }); diff --git a/esp/src/src/Timings.ts b/esp/src/src/Timings.ts index cea27ccbc3d..c7db66385e5 100644 --- a/esp/src/src/Timings.ts +++ b/esp/src/src/Timings.ts @@ -69,17 +69,17 @@ export class Timings { .request({ ScopeFilter: { MaxDepth: 3, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: ["WhenStarted", "TimeElapsed"] } + Properties: ["WhenStarted", "TimeElapsed"] }, ScopeOptions: { IncludeId: true, @@ -210,17 +210,17 @@ export class Timings { this.fetchDetailsNormalizedPromise = Promise.all([this.wu.fetchDetailsMeta(), this.wu.fetchDetailsRaw({ ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: [] } + Properties: [] }, ScopeOptions: { IncludeId: true, diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts index e8ee8e05db8..cc78350ed6d 100644 --- a/esp/src/src/Utility.ts +++ b/esp/src/src/Utility.ts @@ -1303,4 +1303,8 @@ export function wuidToDate(wuid: string): string { export function wuidToTime(wuid: string): string { return `${wuid.substring(10, 12)}:${wuid.substring(12, 14)}:${wuid.substring(14, 16)}`; +} + +export function wuidToDateTime(wuid: string): Date { + return new Date(`${wuidToDate(wuid)}T${wuidToTime(wuid)}Z`); } \ No newline at end of file diff --git a/fs/dafilesrv/dafilesrv.cpp b/fs/dafilesrv/dafilesrv.cpp index 0b991377e75..48e06a24458 100644 --- a/fs/dafilesrv/dafilesrv.cpp +++ b/fs/dafilesrv/dafilesrv.cpp @@ -385,6 +385,7 @@ int main(int argc, const char* argv[]) bool locallisten = false; StringBuffer componentName; + // NB: bare-metal dafilesrv does not have a component specific xml Owned config = loadConfiguration(defaultYaml, argv, "dafilesrv", "DAFILESRV", nullptr, nullptr); Owned keyPairInfo; // NB: not used in containerized mode @@ -516,6 +517,9 @@ int main(int argc, const char* argv[]) Owned _dafileSrvInstance; if (env) { + Owned newConfig = createPTreeFromIPT(config); // clone + IPropertyTree *expert = ensurePTree(newConfig, "expert"); + StringBuffer dafilesrvPath("Software/DafilesrvProcess"); if (componentName.length()) dafilesrvPath.appendf("[@name=\"%s\"]", componentName.str()); @@ -524,6 +528,12 @@ int main(int argc, const char* argv[]) IPropertyTree *daFileSrv = env->queryPropTree(dafilesrvPath); Owned _dafileSrv; + // merge in bare-metal global expert settings + IPropertyTree *globalExpert = nullptr; + globalExpert = env->queryPropTree("Software/Globals"); + if (globalExpert) + synchronizePTree(expert, globalExpert, false, false); + if (daFileSrv) { const char *componentGroupName = daFileSrv->queryProp("@group"); @@ -562,6 +572,12 @@ int main(int argc, const char* argv[]) if (daFileSrv->queryProp("@rowServiceConfiguration")) rowServiceConfiguration = daFileSrv->queryProp("@rowServiceConfiguration"); + // merge in bare-metal dafilesrv component expert settings + IPropertyTree *componentExpert = nullptr; + componentExpert = daFileSrv->queryPropTree("expert"); + if (componentExpert) + synchronizePTree(expert, componentExpert, false, true); + // any overrides by Instance definitions? Owned iter = daFileSrv->getElements("Instance"); ForEach(*iter) @@ -587,8 +603,17 @@ int main(int argc, const char* argv[]) } } } + + // merge in bare-metal dafilesrv instance expert settings + IPropertyTree *instanceExpert = nullptr; + instanceExpert = dafileSrvInstance->queryPropTree("expert"); + if (instanceExpert) + synchronizePTree(expert, instanceExpert, false, true); } + // update config and hook callback with dafilesrv expert PTree + replaceComponentConfig(newConfig, getGlobalConfigSP()); + // bare-metal gets it's certificate info. from environment at the moment, 'keyPairInfo' not used in containerized mode keyPairInfo.set(env->queryPropTree("EnvSettings/Keys")); } diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index c944b0e73d0..9e788fe4447 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -3788,7 +3788,10 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface } IFEflags extraFlags = (IFEflags)extra; // none => nocache for remote (hint) - // can revert to previous behavior with conf file setting "allow_pgcache_flush=false" + // can change this default setting with: + // bare-metal legacy - conf file setting: allow_pgcache_flush=false + // bare-metal - environment.xml dafilesrv expert setting: disableIFileMask=0x1 (IFEnocache) + // containerized - values.yaml dafilesrv expert setting: disableIFileMask: 0x1 (IFEnocache) if (extraFlags == IFEnone) extraFlags = IFEnocache; Owned file = createIFile(name->text); diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 6719b779407..1ccb9d41b5a 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1493,6 +1493,7 @@ Pass in dict with .root, .name, .service, .defaultPort, .selector defined {{- if hasKey $globalServiceInfo "labels" -}}{{- $_ := set $lvars "labels" (merge $lvars.labels $globalServiceInfo.labels) -}}{{- end -}} {{- if hasKey $globalServiceInfo "annotations" -}}{{- $_ := set $lvars "annotations" (merge $lvars.annotations $globalServiceInfo.annotations) -}}{{- end -}} {{- if hasKey $globalServiceInfo "ingress" -}}{{- $_ := set $lvars "ingress" $globalServiceInfo.ingress -}}{{- end -}} + {{- if hasKey $globalServiceInfo "externalTrafficPolicy" -}}{{- $_ := set $lvars "externalTrafficPolicy" $globalServiceInfo.externalTrafficPolicy -}}{{- end -}} {{- if hasKey $globalServiceInfo "loadBalancerSourceRanges" -}}{{- $_ := set $lvars "loadBalancerSourceRanges" $globalServiceInfo.loadBalancerSourceRanges -}}{{- end -}} {{- $_ := set $lvars "type" $globalServiceInfo.type -}} {{- else -}} @@ -1513,6 +1514,7 @@ Pass in dict with .root, .name, .service, .defaultPort, .selector defined {{- end }} {{- end -}} {{- if hasKey .service "ingress" -}}{{- $_ := set $lvars "ingress" .service.ingress -}}{{- end -}} + {{- if hasKey .service "externalTrafficPolicy" -}}{{- $_ := set $lvars "externalTrafficPolicy" .service.externalTrafficPolicy -}}{{- end -}} {{- if hasKey .service "loadBalancerSourceRanges" -}}{{- $_ := set $lvars "loadBalancerSourceRanges" .service.loadBalancerSourceRanges -}}{{- end -}} {{- end }} @@ -1541,6 +1543,9 @@ spec: selector: server: {{ .selector | quote }} type: {{ $lvars.type }} +{{- if $lvars.externalTrafficPolicy }} + externalTrafficPolicy: {{ $lvars.externalTrafficPolicy }} +{{- end }} {{- if $lvars.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{- if ne $lvars.type "LoadBalancer" -}} diff --git a/helm/hpcc/templates/_warnings.tpl b/helm/hpcc/templates/_warnings.tpl index 3fd828069ec..97756916f09 100644 --- a/helm/hpcc/templates/_warnings.tpl +++ b/helm/hpcc/templates/_warnings.tpl @@ -123,27 +123,29 @@ Pass in dict with root and warnings {{- end -}} {{- range $cname, $ctypes := $ctx.components -}} {{- range $id, $component := $ctypes -}} - {{- if and (kindIs "map" $component) (not $component.disabled) -}} - {{- $hasResources := "" -}} - {{- if eq $cname "thor" -}} - {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.managerResources) -}} - {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.workerResources)) $hasResources -}} - {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.eclAgentResources)) $hasResources -}} - {{- else -}} - {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.resources) -}} - {{- end -}} - {{- if not $hasResources -}} - {{- $_ := set $ctx "missingResources" (append $ctx.missingResources ($component.name | default $id)) -}} - {{- end -}} - {{- /* Checks related to components that are used for cost reporting */ -}} - {{- /* (n.b. cpuRate ignored for components other than thor, eclagent and eclccserver)*/ -}} - {{- if has $cname (list "thor" "eclagent" "eclccserver") -}} - {{- if and $ctx.usingDefaultCpuCost (not $component.cost) -}} - {{- $_ := set $ctx "defaultCpuRateComponents" (append $ctx.defaultCpuRateComponents $component.name) -}} + {{- if (kindIs "map" $component) -}} + {{- if (not $component.disabled) -}} + {{- $hasResources := "" -}} + {{- if eq $cname "thor" -}} + {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.managerResources) -}} + {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.workerResources)) $hasResources -}} + {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.eclAgentResources)) $hasResources -}} + {{- else -}} + {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.resources) -}} {{- end -}} - {{- /* Components that are used for cost reporting require resources: warn if resources missing*/ -}} {{- if not $hasResources -}} - {{- $_ := set $ctx "missingResourcesForCosts" (append $ctx.missingResourcesForCosts ($component.name | default $id)) -}} + {{- $_ := set $ctx "missingResources" (append $ctx.missingResources ($component.name | default $id)) -}} + {{- end -}} + {{- /* Checks related to components that are used for cost reporting */ -}} + {{- /* (n.b. cpuRate ignored for components other than thor, eclagent and eclccserver)*/ -}} + {{- if has $cname (list "thor" "eclagent" "eclccserver") -}} + {{- if and $ctx.usingDefaultCpuCost (not $component.cost) -}} + {{- $_ := set $ctx "defaultCpuRateComponents" (append $ctx.defaultCpuRateComponents $component.name) -}} + {{- end -}} + {{- /* Components that are used for cost reporting require resources: warn if resources missing*/ -}} + {{- if not $hasResources -}} + {{- $_ := set $ctx "missingResourcesForCosts" (append $ctx.missingResourcesForCosts ($component.name | default $id)) -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 1355263d5ba..b458e708383 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -67,7 +67,7 @@ true {{- end -}} {{- end }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ $dali.name | quote }} spec: @@ -77,6 +77,10 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 template: metadata: labels: @@ -97,6 +101,13 @@ spec: {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }} {{- end }} spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: dali + topologyKey: "kubernetes.io/hostname" {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $dali.name "type" "dali") | indent 6 }} serviceAccountName: "hpcc-dali" terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds | default 3600 }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 1cab2f5cdd1..da0207f7fc8 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -43,6 +43,8 @@ data: {{- include "hpcc.generateLoggingConfig" . | indent 6 }} {{- include "hpcc.generateTracingConfig" . | indent 6 }} {{ include "hpcc.generateVaultConfig" . | indent 6 }} + queues: +{{ include "hpcc.generateConfigMapQueues" .root | indent 6 }} global: {{ include "hpcc.generateGlobalConfigMap" .root | indent 6 }} {{- if not .me.useChildProcesses }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 456c665985e..863e7cb3e4c 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -28,10 +28,10 @@ Pass in dict with root and me {{- define "hpcc.thorConfigMap" -}} {{- $eclAgentType := .me.eclAgentType | default "hthor" }} {{- $hthorName := printf "%s-%s" .me.name $eclAgentType }} -{{- $eclAgentScope := dict "name" .eclAgentName "type" $eclAgentType "useChildProcesses" .eclAgentUseChildProcesses "replicas" .eclAgentReplicas "maxActive" .me.maxJobs | merge (pick .me "keepJobs" "logging" "tracing" "auxQueues") }} -{{- $thorAgentScope := dict "name" .thorAgentName "replicas" .thorAgentReplicas "maxActive" .me.maxGraphs | merge (pick .me "keepJobs" "logging" "tracing") }} +{{- $eclAgentScope := dict "name" .eclAgentName "type" $eclAgentType "useChildProcesses" .eclAgentUseChildProcesses "replicas" .eclAgentReplicas "maxActive" .me.maxJobs | merge (pick .me "keepJobs" "logging" "tracing" "auxQueues") ((hasKey .me "holdAgent") | ternary (dict "hold" .me.holdAgent) dict) }} +{{- $thorAgentScope := dict "name" .thorAgentName "replicas" .thorAgentReplicas "maxActive" .me.maxGraphs | merge (pick .me "keepJobs" "logging" "tracing") ((hasKey .me "holdThorAgent") | ternary (dict "hold" .me.holdThorAgent) dict) }} {{- $eclAgentResources := .me.eclAgentResources | default dict -}} -{{- $hthorScope := dict "name" $hthorName "jobMemorySectionName" "eclAgentMemory" | merge (pick .me "multiJobLinger" "maxGraphStartupTime" "logging" "tracing") | merge (dict "resources" (deepCopy $eclAgentResources)) }} +{{- $hthorScope := dict "name" $hthorName "jobMemorySectionName" "eclAgentMemory" | merge (pick .me "multiJobLinger" "maxGraphStartupTime" "logging" "tracing") | merge (dict "resources" (deepCopy $eclAgentResources)) ((hasKey .me "holdWorkflow") | ternary (dict "hold" .me.holdWorkflow) dict)}} {{- $thorScope := omit .me "eclagent" "thoragent" "hthor" "env" "eclAgentResources" "eclAgentUseChildProcesses" "eclAgentReplicas" "thorAgentReplicas" "eclAgentType" }} {{- $misc := .root.Values.global.misc | default dict }} {{- $postJobCommand := $misc.postJobCommand | default "" }} @@ -61,6 +61,8 @@ data: {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }} {{- include "hpcc.generateTracingConfig" (dict "root" .root "me" $hthorScope ) | indent 6 }} {{ include "hpcc.generateVaultConfig" . | indent 6 }} + queues: +{{ include "hpcc.generateConfigMapQueues" .root | indent 6 }} eclagent: # main agent Q handler {{ toYaml (omit $eclAgentScope "logging" "tracing") | indent 6 }} {{- include "hpcc.generateLoggingConfig" (dict "root" .root "me" $eclAgentScope) | indent 6 }} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 768ff3eab98..b42f1f297d3 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -404,6 +404,11 @@ "ingress": { "$ref" : "#/definitions/ingress" }, + "externalTrafficPolicy": { + "type": "string", + "enum": ["Cluster", "Local"], + "description": "Route clients to all (Cluster), or local VM only (Local). Preserves client source IPs." + }, "loadBalancerSourceRanges": { "type": "array", "items": { "type": "string" }, @@ -1292,6 +1297,11 @@ "type": "object", "additionalProperties": { "type": "string" } }, + "externalTrafficPolicy": { + "type": "string", + "enum": ["Cluster", "Local"], + "description": "Route clients to all (Cluster), or local VM only (Local). Preserves client source IPs." + }, "loadBalancerSourceRanges": { "type": "array", "items": { "type": "string" }, diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml index d0b8eca46dc..b6a6d05ccba 100644 --- a/helm/hpcc/values.yaml +++ b/helm/hpcc/values.yaml @@ -115,6 +115,8 @@ global: type: LoadBalancer ingress: - {} + ## route traffic to local VM only. Also preserves source IPs + #externalTrafficPolicy: Local ## CIDRS allowed to access this service. #loadBalancerSourceRanges: [1.2.3.4/32, 5.6.7.8/32] @@ -434,10 +436,13 @@ dafilesrv: service: servicePort: 7300 visibility: cluster - + # example expert section. A disableIFileMask of 0x1 will disable the IFEnocache option + # expert: + # disableIFileMask: 0x1 dali: - name: mydali + #hold: true # (for developers, can be used on any component). Component will enter a hold loop at config load time. auth: none services: # internal house keeping services coalescer: @@ -642,6 +647,8 @@ esp: # url: "http://abc.com/def?g=1" ## CIDRS allowed to access this service. #loadBalancerSourceRanges: [1.2.3.4/32, 5.6.7.8/32] + ## route traffic to local VM only. Also preserves source IPs + #externalTrafficPolicy: Local # Increase maxRequestEntityLength when query deployments (or similar actions) start to fail because they surpass the maximum size # default for EclWatch is 60M, default for other services is 8M #maxRequestEntityLength: 70M @@ -810,6 +817,10 @@ roxie: ## [worker/manager]Memory.thirdParty. thor: - name: thor + #hold: true # (for developers, can be used on any component). Component will enter a hold loop at config load time. + #holdSlave: 0 # (for developers). Similar to hold, but 0 = master, 1-N = worker #. + #holdAgent: true # (for developers). Similar to hold, but for this Thor's eclagent. + #holdThorAgent: true # (for developers). Similar to hold, but for this Thor's thoragent. prefix: thor numWorkers: 2 maxJobs: 4 diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index f51aee989da..77b2dd66d3a 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -451,6 +451,8 @@ extern unsigned agentQueryReleaseDelaySeconds; extern unsigned coresPerQuery; extern unsigned cacheReportPeriodSeconds; +extern stat_type minimumInterestingActivityCycles; + extern StringBuffer logDirectory; extern StringBuffer pluginDirectory; @@ -591,6 +593,9 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface bool blind; mutable bool aborted; mutable CIArrayOf log; + static constexpr const unsigned MaxSlowActivities = 5; + mutable unsigned slowestActivityIds[MaxSlowActivities] = {}; + mutable stat_type slowestActivityTimes[MaxSlowActivities] = {}; private: Owned activeSpan = getNullSpan(); ContextLogger(const ContextLogger &); // Disable copy constructor @@ -692,11 +697,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface ctxTraceLevel = _level; } - StringBuffer &getStats(StringBuffer &s) const - { - CriticalBlock block(statsCrit); - return stats.toStr(s); - } + StringBuffer &getStats(StringBuffer &s) const; virtual bool isIntercepted() const { @@ -720,18 +721,8 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const - { - if (from.isThreadSafeMergeSource()) - { - stats.merge(from); - } - else - { - CriticalBlock block(statsCrit); - stats.merge(from); - } - } + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const; + virtual void gatherStats(CRuntimeStatisticCollection & merged) const override { merged.merge(stats); diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 22ec7c6f25b..dc6d800b848 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1314,9 +1314,9 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext logctx.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const override + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { - logctx.mergeStats(from); + logctx.mergeStats(activityId, from); } virtual void gatherStats(CRuntimeStatisticCollection & merged) const override @@ -2379,7 +2379,7 @@ class CAgentContext : public CRoxieContextBase { // NOTE: This is needed to ensure that owned activities are destroyed BEFORE I am, // to avoid pure virtual calls when they come to call noteProcessed() - logctx.mergeStats(globalStats); + logctx.mergeStats(0, globalStats); if (factory) factory->mergeStats(logctx); childGraphs.releaseAll(); @@ -2611,7 +2611,7 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon void doPostProcess() { - logctx.mergeStats(globalStats); + logctx.mergeStats(0, globalStats); logctx.setStatistic(StTimeTotalExecute, elapsedTimer.elapsedNs()); if (factory) { diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index d38624421a1..85fd1241a4e 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -945,6 +945,64 @@ extern void updateAffinity(unsigned __int64 affinity) //-------------------------------------------------------------------------------------------------------------------- +StringBuffer & ContextLogger::getStats(StringBuffer &s) const +{ + CriticalBlock block(statsCrit); + stats.toStr(s); + + if (slowestActivityIds[0]) + { + StringBuffer ids; + StringBuffer times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowestActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowestActivityIds[i]); + formatStatistic(times, cycle_to_nanosec(slowestActivityTimes[i]), SMeasureTimeNs); + } + s.appendf(", slowestActivities={ ids=[%s] times=[%s] }", ids.str(), times.str()); + } + return s; +} + + +void ContextLogger::mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const +{ + CLeavableCriticalBlock block(statsCrit, !from.isThreadSafeMergeSource()); + + stats.merge(from); + + //Record the times of the slowest N activities + if (activityId) + { + stat_type localTime = from.getStatisticValue(StCycleLocalExecuteCycles); + if (localTime >= minimumInterestingActivityCycles) + { + if (localTime > slowestActivityTimes[MaxSlowActivities-1]) + { + unsigned pos = MaxSlowActivities-1; + while (pos > 0) + { + if (localTime <= slowestActivityTimes[pos-1]) + break; + slowestActivityIds[pos] = slowestActivityIds[pos-1]; + slowestActivityTimes[pos] = slowestActivityTimes[pos-1]; + pos--; + } + slowestActivityIds[pos] = activityId; + slowestActivityTimes[pos] = localTime; + } + } + } +} + void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends) { if (activeSpan->isRecording()) @@ -957,6 +1015,29 @@ void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned StringBuffer prefix(""); stats.exportToSpan(activeSpan, prefix); + + if (slowestActivityIds[0]) + { + //Even better if these were exported as arrays - needs extensions to our api + //Not commoned up with the code above because it is likely to change to arrays in the future. + StringBuffer ids; + StringBuffer times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowestActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowestActivityIds[i]); + times.append(cycle_to_nanosec(slowestActivityTimes[i])); + } + setSpanAttribute("slowest_activities.ids", ids); + setSpanAttribute("slowest_activities.times", times); + } } } diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index b686d010df3..31c05a1080a 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -208,6 +208,7 @@ unsigned __int64 minFreeDiskSpace = 1024 * 0x100000; // default to 1 GB unsigned socketCheckInterval = 5000; unsigned cacheReportPeriodSeconds = 5*60; +stat_type minimumInterestingActivityCycles; StringBuffer logDirectory; StringBuffer pluginDirectory; @@ -1318,6 +1319,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) unsigned __int64 affinity = topology->getPropInt64("@affinity", 0); updateAffinity(affinity); + unsigned __int64 minimumInterestingActivityMs = topology->getPropInt64("@minimumInterestingActivityMs", 10); + minimumInterestingActivityCycles = nanosec_to_cycle(minimumInterestingActivityMs * 1'000'000); + minFreeDiskSpace = topology->getPropInt64("@minFreeDiskSpace", (1024 * 0x100000)); // default to 1 GB mtu_size = topology->getPropInt("@mtuPayload", 0); if (mtu_size) diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 8bad56d5e2d..45a76675e46 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -176,55 +176,55 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface // void set(IRoxieAgentContext * _ctx) { ctx = _ctx; } - virtual ICodeContext *queryCodeContext() + virtual ICodeContext *queryCodeContext() override { return ctx->queryCodeContext(); } - virtual void checkAbort() + virtual void checkAbort() override { ctx->checkAbort(); } - virtual unsigned checkInterval() const + virtual unsigned checkInterval() const override { return ctx->checkInterval(); } - virtual void notifyAbort(IException *E) + virtual void notifyAbort(IException *E) override { ctx->notifyAbort(E); } - virtual void notifyException(IException *E) + virtual void notifyException(IException *E) override { ctx->notifyException(E); } - virtual void throwPendingException() + virtual void throwPendingException() override { ctx->throwPendingException(); } - virtual IActivityGraph * queryChildGraph(unsigned id) + virtual IActivityGraph * queryChildGraph(unsigned id) override { return ctx->queryChildGraph(id); } - virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph) + virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph) override { ctx->noteChildGraph(id, childGraph) ; } - virtual IRowManager &queryRowManager() + virtual IRowManager &queryRowManager() override { return ctx->queryRowManager(); } - virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const + virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const override { ctx->noteStatistic(kind, value); } - virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const + virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const override { ctx->setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { - ctx->mergeStats(from); + ctx->mergeStats(activityId, from); } - virtual StringBuffer &getStats(StringBuffer &ret) const + virtual StringBuffer &getStats(StringBuffer &ret) const override { return ctx->getStats(ret); } @@ -236,7 +236,7 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->recordStatistics(progress); } - virtual bool collectingDetailedStatistics() const + virtual bool collectingDetailedStatistics() const override { return ctx->collectingDetailedStatistics(); } @@ -248,15 +248,15 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->CTXLOGa(category, cat, code, prefix, text); } - virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) + virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const override __attribute__((format(printf,5,0))) { ctx->logOperatorExceptionVA(E, file, line, format, args); } - virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute__((format(printf,6,0))) + virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const override __attribute__((format(printf,6,0))) { ctx->CTXLOGaeva(E, file, line, prefix, format, args); } - virtual void CTXLOGl(LogItem *log) const + virtual void CTXLOGl(LogItem *log) const override { ctx->CTXLOGl(log); } @@ -264,15 +264,15 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->getLogPrefix(ret); } - virtual unsigned queryTraceLevel() const + virtual unsigned queryTraceLevel() const override { return ctx->queryTraceLevel(); } - virtual bool isIntercepted() const + virtual bool isIntercepted() const override { return ctx->isIntercepted(); } - virtual bool isBlind() const + virtual bool isBlind() const override { return ctx->isBlind(); } @@ -300,7 +300,7 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->setSpanAttribute(name, value); } - virtual const char *queryGlobalId() const + virtual const char *queryGlobalId() const override { return ctx->queryGlobalId(); } @@ -308,83 +308,83 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->queryCallerId(); } - virtual const char *queryLocalId() const + virtual const char *queryLocalId() const override { return ctx->queryLocalId(); } - virtual const QueryOptions &queryOptions() const + virtual const QueryOptions &queryOptions() const override { return ctx->queryOptions(); } - virtual void addAgentsReplyLen(unsigned len, unsigned duplicates, unsigned resends) + virtual void addAgentsReplyLen(unsigned len, unsigned duplicates, unsigned resends) override { ctx->addAgentsReplyLen(len, duplicates, resends); } - virtual const char *queryAuthToken() + virtual const char *queryAuthToken() override { return ctx->queryAuthToken(); } - virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser) + virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser) override { return ctx->resolveLFN(filename, isOpt, isPrivilegedUser); } - virtual IRoxieWriteHandler *createWriteHandler(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser) + virtual IRoxieWriteHandler *createWriteHandler(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser) override { return ctx->createWriteHandler(filename, overwrite, extend, clusters, isPrivilegedUser); } - virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser) + virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser) override { ctx->onFileCallback(header, lfn, isOpt, isLocal, isPrivilegedUser); } - virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity) + virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity) override { return ctx->getLibraryGraph(extra, parentActivity); } - virtual IProbeManager *queryProbeManager() const + virtual IProbeManager *queryProbeManager() const override { return ctx->queryProbeManager(); } - virtual IDebuggableContext *queryDebugContext() const + virtual IDebuggableContext *queryDebugContext() const override { return ctx->queryDebugContext(); } - virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) + virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) override { ctx->printResults(output, name, sequence); } - virtual void setWUState(WUState state) + virtual void setWUState(WUState state) override { ctx->setWUState(state); } - virtual bool checkWuAborted() + virtual bool checkWuAborted() override { return ctx->checkWuAborted(); } - virtual IWorkUnit *updateWorkUnit() const + virtual IWorkUnit *updateWorkUnit() const override { return ctx->updateWorkUnit(); } - virtual IConstWorkUnit *queryWorkUnit() const + virtual IConstWorkUnit *queryWorkUnit() const override { return ctx->queryWorkUnit(); } - virtual IRoxieServerContext *queryServerContext() + virtual IRoxieServerContext *queryServerContext() override { return ctx->queryServerContext(); } - virtual IWorkUnitRowReader *getWorkunitRowReader(const char *wuid, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) + virtual IWorkUnitRowReader *getWorkunitRowReader(const char *wuid, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) override { return ctx->getWorkunitRowReader(wuid, name, sequence, xmlTransformer, rowAllocator, isGrouped); } - virtual IEngineRowAllocator *getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, roxiemem::RoxieHeapFlags flags) const + virtual IEngineRowAllocator *getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, roxiemem::RoxieHeapFlags flags) const override { return ctx->getRowAllocatorEx(meta, activityId, flags); } - virtual void noteLibrary(IQueryFactory *library) + virtual void noteLibrary(IQueryFactory *library) override { ctx->noteLibrary(library); } - virtual const CRuntimeStatisticCollection & queryStats() const + virtual const CRuntimeStatisticCollection & queryStats() const override { return ctx->queryStats(); } @@ -681,7 +681,7 @@ class CRoxieServerActivityFactoryBase : public CActivityFactory, implements IRox throwUnexpected(); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(const CRuntimeStatisticCollection &from) const override { CActivityFactory::mergeStats(from); } @@ -1265,7 +1265,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl //Updates the query summary statistics if (ctx) - ctx->queryCodeContext()->queryContextLogger().mergeStats(mergedStats); + ctx->queryCodeContext()->queryContextLogger().mergeStats(activityId, mergedStats); ForEachItemIn(i, childGraphs) childGraphs.item(i).gatherStatistics(statsBuilder); @@ -1287,7 +1287,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { stats.deserializeMerge(buf); } - virtual void mergeStats(const CRuntimeStatisticCollection & childStats) + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection & childStats) { CriticalBlock b(statscrit); stats.merge(childStats); @@ -1383,7 +1383,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const { stats.merge(from); } @@ -5212,7 +5212,7 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie CRuntimeStatisticCollection childStats(allStatistics); childStats.deserialize(buf); //activity.queryContext()->mergeActivityStats(childStats, graphId, childId); - activity.mergeStats(childStats); + activity.mergeStats(0, childStats); } } ReleaseRoxieRow(logInfo); diff --git a/roxie/ccd/ccdserver.hpp b/roxie/ccd/ccdserver.hpp index 402436abdc2..a1a5c7e452b 100644 --- a/roxie/ccd/ccdserver.hpp +++ b/roxie/ccd/ccdserver.hpp @@ -198,7 +198,7 @@ interface IRoxieServerActivity : extends IActivityBase virtual ThorActivityKind getKind() const = 0; virtual const IRoxieContextLogger &queryLogCtx() const = 0; virtual void mergeStats(MemoryBuffer &stats) = 0; - virtual void mergeStats(const CRuntimeStatisticCollection & childStats) = 0; + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection & childStats) = 0; virtual ISectionTimer * registerTimer(unsigned activityId, const char * name) = 0; virtual IEngineRowAllocator * createRowAllocator(IOutputMetaData * metadata) = 0; virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const = 0; diff --git a/system/jhtree/jhtree.cpp b/system/jhtree/jhtree.cpp index 86c93a1de30..292f2e1dcd5 100644 --- a/system/jhtree/jhtree.cpp +++ b/system/jhtree/jhtree.cpp @@ -1244,8 +1244,18 @@ const CJHTreeNode *CDiskKeyIndex::loadNode(cycle_t * fetchCycles, offset_t pos, nodesLoaded++; if (!useIO) useIO = io; unsigned nodeSize = keyHdr->getNodeSize(); + + //Use alloca() to allocate a buffer on the stack if the node size is small enough. + //Often the data could be read directly from the input files's buffer and not even be copied. + //Should we have a io->peek(pos, size) which returns a pointer if supported? + constexpr const size_t maxStackSize = 8192; // Default node size MemoryAttr ma; - char *nodeData = (char *) ma.allocate(nodeSize); + char *nodeData; + if (nodeSize <= maxStackSize) + nodeData = (char *) alloca(nodeSize); + else + nodeData = (char *) ma.allocate(nodeSize); + assertex(nodeData); CCycleTimer fetchTimer(fetchCycles != nullptr); if (useIO->read(pos, nodeSize, nodeData) != nodeSize) diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 5df2af3d697..d4f3689163c 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -102,6 +102,10 @@ static inline bool isPCFlushAllowed(); static char ShareChar='$'; +// defaults +static IFEflags expertEnableIFileFlagsMask = IFEnone; +static IFEflags expertDisableIFileFlagsMask = IFEnone; + bool isShareChar(char c) { return (c==ShareChar)||(c=='$'); @@ -2055,11 +2059,17 @@ CFileIO::CFileIO(HANDLE handle, IFOmode _openmode, IFSHmode _sharemode, IFEflags sharemode = _sharemode; openmode = _openmode; extraFlags = _extraFlags; + + // leave for compatibility if (extraFlags & IFEnocache) if (!isPCFlushAllowed()) extraFlags = static_cast(extraFlags & ~IFEnocache); + + extraFlags = static_cast(extraFlags | expertEnableIFileFlagsMask); + extraFlags = static_cast(extraFlags & ~expertDisableIFileFlagsMask); + #ifdef CFILEIOTRACE - DBGLOG("CFileIO::CfileIO(%d,%d,%d,%d)", handle, _openmode, _sharemode, _extraFlags); + DBGLOG("CFileIO::CfileIO(%d,%d,%d,%d)", handle, _openmode, _sharemode, extraFlags); #endif } @@ -7846,7 +7856,8 @@ bool hasGenericFiletypeName(const char * name) ///--------------------------------------------------------------------------------------------------------------------- // Cache/update plane attributes settings -static unsigned planeAttributeCBId = 0; +static unsigned jFileHookId = 0; + static const std::array planeAttributeTypeStrings = { "blockedFileIOKB", @@ -7870,14 +7881,26 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) values[BlockedSequentialIO] = plane.getPropInt(("@" + std::string(planeAttributeTypeStrings[BlockedSequentialIO])).c_str()) * 1024; values[BlockedRandomIO] = plane.getPropInt(("@" + std::string(planeAttributeTypeStrings[BlockedRandomIO])).c_str()) * 1024; } + + // reset defaults + expertEnableIFileFlagsMask = IFEnone; + expertDisableIFileFlagsMask = IFEnone; + + StringBuffer fileFlagsStr; + if (getComponentConfigSP()->getProp("expert/@enableIFileMask", fileFlagsStr) || getGlobalConfigSP()->getProp("expert/@enableIFileMask", fileFlagsStr)) + expertEnableIFileFlagsMask = (IFEflags)strtoul(fileFlagsStr, NULL, 0); + + if (getComponentConfigSP()->getProp("expert/@disableIFileMask", fileFlagsStr.clear()) || getGlobalConfigSP()->getProp("expert/@disableIFileMask", fileFlagsStr)) + expertDisableIFileFlagsMask = (IFEflags)strtoul(fileFlagsStr, NULL, 0); }; - planeAttributeCBId = installConfigUpdateHook(updateFunc, true); + jFileHookId = installConfigUpdateHook(updateFunc, true); + return true; } MODULE_EXIT() { - removeConfigUpdateHook(planeAttributeCBId); + removeConfigUpdateHook(jFileHookId); } const char *getPlaneAttributeString(PlaneAttributeType attr) diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index a38601379de..ba13f2b9276 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -2779,7 +2779,7 @@ class DummyLogCtx : implements IContextLogger virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const { } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const { } virtual unsigned queryTraceLevel() const diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 322b58c444e..5df0e05a343 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1272,7 +1272,7 @@ interface jlib_decl IContextLogger : extends IInterface virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const = 0; virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const = 0; - virtual void mergeStats(const CRuntimeStatisticCollection &from) const = 0; + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const = 0; virtual unsigned queryTraceLevel() const = 0; virtual const char *queryGlobalId() const = 0; diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 625e1a1652d..de2d4ef3cb0 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -303,7 +303,8 @@ void CLZWCompressor::open(void *buf,size32_t max) outbuf = malloc(bufalloc); } outBufMb = NULL; - assertex(max>SAFETY_MARGIN+sizeof(size32_t)); // minimum required + if (max<=SAFETY_MARGIN+sizeof(size32_t)) // minimum required + throw makeStringException(0, "CLZWCompressor: target buffer too small"); maxlen=max-SAFETY_MARGIN; initCommon(); } @@ -1386,7 +1387,8 @@ class jlib_decl CRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - assertex(max>2+sizeof(size32_t)*2); // minimum required (actually will need enough for recsize so only a guess) + if (max<=2+sizeof(size32_t)*2) // minimum required (actually will need enough for recsize so only a guess) + throw makeStringException(0, "CRDiffCompressor: target buffer too small"); initCommon(); remaining = max-outlen; } @@ -1669,7 +1671,8 @@ class jlib_decl CRandRDiffCompressor : public ICompressor, public CInterface outbuf = malloc(bufalloc); } outBufMb = NULL; - assertex(max>MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3); // hopefully a lot bigger! + if (max<=MIN_RRDHEADER_SIZE+sizeof(unsigned short)+3) // hopefully a lot bigger! + throw makeStringException(0, "CRandRDiffCompressor: target buffer too small"); initCommon(); } diff --git a/testing/helm/run.sh b/testing/helm/run.sh index d3f2ef17367..27db917d9f6 100755 --- a/testing/helm/run.sh +++ b/testing/helm/run.sh @@ -78,6 +78,7 @@ if type kube-score >/dev/null 2> /dev/null; then --ignore-container-cpu-limit \ --ignore-container-memory-limit \ --ignore-test deployment-has-poddisruptionbudget \ + --ignore-test statefulset-has-poddisruptionbudget \ - >results.txt 2>errors.txt if [ $? -ne 0 ] then diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 25723a2d62e..e935b67cb0e 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -4083,6 +4083,116 @@ CPPUNIT_TEST_SUITE_REGISTRATION( JLibOpensslAESTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibOpensslAESTest, "JLibOpensslAESTest" ); #endif +class TopNStressTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(TopNStressTest); + CPPUNIT_TEST(testTop5a); + CPPUNIT_TEST(testTop5b); + CPPUNIT_TEST_SUITE_END(); + + struct x + { + static constexpr const unsigned MaxSlowActivities = 5; + mutable unsigned slowActivityIds[MaxSlowActivities] = {}; + mutable stat_type slowActivityTimes[MaxSlowActivities] = {}; + unsigned cnt = 0; + + __attribute__((noinline)) + void noteTime(unsigned activityId, stat_type localTime) + { + if (localTime > slowActivityTimes[MaxSlowActivities-1]) + { + unsigned pos = MaxSlowActivities-1; + while (pos > 0) + { + if (localTime <= slowActivityTimes[pos-1]) + break; + slowActivityIds[pos] = slowActivityIds[pos-1]; + slowActivityTimes[pos] = slowActivityTimes[pos-1]; + pos--; + } + slowActivityIds[pos] = activityId; + slowActivityTimes[pos] = localTime; + cnt++; + } + } + __attribute__((noinline)) + void noteTime2(unsigned activityId, stat_type localTime) + { + if (localTime > slowActivityTimes[0]) + { + unsigned pos = 1; + while (pos < MaxSlowActivities) + { + if (localTime <= slowActivityTimes[pos]) + break; + slowActivityIds[pos-1] = slowActivityIds[pos]; + slowActivityTimes[pos-1] = slowActivityTimes[pos]; + pos++; + } + slowActivityIds[pos-1] = activityId; + slowActivityTimes[pos-1] = localTime; + cnt++; + } + } + void report(__uint64 elapsedNs) + { + StringBuffer ids, times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowActivityIds[i]); + formatStatistic(times, cycle_to_nanosec(slowActivityTimes[i]), SMeasureTimeNs); + } + DBGLOG("SlowActivities={ ids=[%s] times=[%s] } in %llu (x%u)", ids.str(), times.str(), elapsedNs, cnt); + + } + }; + + static constexpr const unsigned NumIters = 5000; + + void testTop5a() + { + CCycleTimer timer; + stat_type value = 0; + x tally; + for (unsigned activityId = 1; activityId <= NumIters; activityId++) + { + value = value * 0x100000001b3ULL + 99; + tally.noteTime(activityId, value); + } + + __uint64 elapsedNs = timer.elapsedNs(); + tally.report(elapsedNs); + } + void testTop5b() + { + CCycleTimer timer; + stat_type value = 0; + x tally; + for (unsigned activityId = 1; activityId <= NumIters; activityId++) + { + value = value * 0x100000001b3ULL + 99; + tally.noteTime2(activityId, value); + } + + __uint64 elapsedNs = timer.elapsedNs(); + tally.report(elapsedNs); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( TopNStressTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( TopNStressTest, "TopNStressTest" ); + class JLibSecretsTest : public CppUnit::TestFixture { public: @@ -4408,8 +4518,6 @@ class JLibSecretsTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( JLibSecretsTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibSecretsTest, "JLibSecretsTest" ); - - class JLibStringTest : public CppUnit::TestFixture { public: diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index feb7591c556..5f543423889 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -406,8 +406,8 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl { Owned sendBucket = _sendBucket.getClear(); size32_t writerTotalSz = 0; - size32_t remoteSendSz = 0; - unsigned remoteRowCount = 0; + offset_t remoteSendSz = 0; + offset_t remoteRowCount = 0; CMessageBuffer msg; while (!owner.aborted) { diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp index 0ffb609dcf4..8844a99b7f3 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp @@ -1768,7 +1768,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem CRuntimeStatisticCollection statsDelta(jhtreeCacheStatistics); statsDelta.deserialize(mb); CStatsContextLogger * contextLogger(contextLoggers[selected]); - contextLogger->mergeStats(statsDelta); + contextLogger->mergeStats(activity.queryActivityId(), statsDelta); if (received == numRows) break; } diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 417ccda504f..f2603ea7c45 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -211,8 +211,19 @@ class CWriteIntercept : public CSimpleInterface rwFlags |= rw_compress; rwFlags |= spillCompInfo; compressedOverflowFile = true; - compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, DEFAULT_SORT_COMPBLKSZ); - ActPrintLog(&activity, "Creating compressed merged overflow file (block size = %u)", compBlkSz); + + /* + * NB: HPCC-29385 Changed the way that compressed files are decompressed, so they are only decompressed one + * compression buffer at a time. For LZ4 this means they can only ever expand to the compressed block size (typically 1MB) + * rather than 10s of times that space. + * LZW could still expand many times its block size, but the default for LZW is already 64K which is low enough. + * Therefore we default to 0 here, which causes createRowWriter to use the default block size for the compressor type. + */ + compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, 0); + if (compBlkSz) + ActPrintLog(&activity, "Creating compressed merged overflow file (block size = %u)", compBlkSz); + else + ActPrintLog(&activity, "Creating compressed merged overflow file"); } } diff --git a/thorlcr/thorutil/thbufdef.hpp b/thorlcr/thorutil/thbufdef.hpp index bb61498962c..a5291a30364 100644 --- a/thorlcr/thorutil/thbufdef.hpp +++ b/thorlcr/thorutil/thbufdef.hpp @@ -54,7 +54,6 @@ #define EXCESSIVE_PARALLEL_THRESHHOLD (0x500000) // 5MB #define LOOP_SMART_BUFFER_SIZE (0x100000*12) // 12MB #define LOCALRESULT_BUFFER_SIZE (0x100000*10) // 10MB -#define DEFAULT_SORT_COMPBLKSZ (0x10000) // 64K #define DEFAULT_KEYNODECACHEMB 10 #define DEFAULT_KEYLEAFCACHEMB 50 diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 65d11882106..3677e86647c 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -1862,9 +1862,19 @@ class CThorRowCollectorBase : public CSpillable * if there are a lot of spill files, the merge opens them all and causes excessive * memory usage. */ - size32_t compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, DEFAULT_SORT_COMPBLKSZ); - ActPrintLog(&activity, thorDetailedLogLevel, "%sSpilling will use compressed block size = %u", tracingPrefix.str(), compBlkSz); - spillableRows.setCompBlockSize(compBlkSz); + /* + * However.... HPCC-29385 Changed the way that compressed files are decompressed, so they are only decompressed one + * compression buffer at a time. For LZ4 this means they can only ever expand to the compressed block size (typically 1MB) + * rather than 10s of times that space. + * LZW could still expand many times its block size, but the default for LZW is already 64K which is low enough. + * Therefore only set the compress block size if it has been explicitly set. + */ + size32_t compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, 0); + if (compBlkSz) + { + ActPrintLog(&activity, thorDetailedLogLevel, "%sSpilling will use compressed block size = %u", tracingPrefix.str(), compBlkSz); + spillableRows.setCompBlockSize(compBlkSz); + } } } ~CThorRowCollectorBase()