From c74f3ed692a3b3a88e6e45ef972ce8d83a5c6f90 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 20 May 2024 17:35:19 +0100 Subject: [PATCH 01/18] HPCC-31848 Run workunit analyser after each graph and on failure Signed-off-by: Gavin Halliday --- common/wuanalysis/anawu.cpp | 29 +++++++++++++++++----------- common/wuanalysis/anawu.hpp | 2 +- ecl/eclagent/eclagent.cpp | 38 ++++++++++++++++++++++++++++++++----- ecl/eclagent/eclagent.ipp | 2 ++ ecl/eclagent/eclgraph.cpp | 11 ++++++++++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/common/wuanalysis/anawu.cpp b/common/wuanalysis/anawu.cpp index a8b7aac1c96..79aa20c9495 100644 --- a/common/wuanalysis/anawu.cpp +++ b/common/wuanalysis/anawu.cpp @@ -155,7 +155,7 @@ class WorkunitAnalyserBase public: WorkunitAnalyserBase(); - void analyse(IConstWorkUnit * wu); + void analyse(IConstWorkUnit * wu, const char * optGraph); WuScope * getRootScope() { return LINK(root); } protected: @@ -1282,10 +1282,17 @@ WorkunitAnalyserBase::WorkunitAnalyserBase() : root(new WuScope("", nullptr)) { } -void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu) +void WorkunitAnalyserBase::analyse(IConstWorkUnit * wu, const char * optGraph) { WuScopeFilter filter; filter.addOutputProperties(PTstatistics).addOutputProperties(PTattributes); + if (optGraph) + { + //Only include the specified graph, and include everything that matches below that graph + filter.addScopeType(SSTgraph); + filter.addId(optGraph); + filter.setIncludeNesting((unsigned)-1); + } filter.finishedFilter(); collateWorkunitStats(wu, filter); root->connectActivities(); @@ -2079,11 +2086,11 @@ void WorkunitStatsAnalyser::traceDependencies() //--------------------------------------------------------------------------------------------------------------------- -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs) +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs) { WorkunitRuleAnalyser analyser; analyser.applyConfig(options, wu); - analyser.analyse(wu); + analyser.analyse(wu, optGraph); analyser.applyRules(); analyser.update(wu, costPerMs); } @@ -2092,7 +2099,7 @@ void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costRate, { WorkunitRuleAnalyser analyser; analyser.applyConfig(nullptr, wu); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.applyRules(); analyser.print(); if (updatewu) @@ -2114,7 +2121,7 @@ void analyseActivity(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.reportActivity(args); } @@ -2123,7 +2130,7 @@ void analyseDependencies(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.spotCommonPath(args); @@ -2137,7 +2144,7 @@ void analyseOutputDependencyGraph(IConstWorkUnit * wu, IPropertyTree * cfg) { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceDependencies(); @@ -2147,7 +2154,7 @@ void analyseCriticalPath(IConstWorkUnit * wu, IPropertyTree * cfg, const StringA { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.adjustTimestamps(); analyser.calcDependencies(); analyser.traceCriticalPaths(args); @@ -2157,7 +2164,7 @@ void analyseHotspots(IConstWorkUnit * wu, IPropertyTree * cfg, const StringArray { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); const char * rootScope = nullptr; if (args.ordinality()) @@ -2177,7 +2184,7 @@ void analyseHotspots(WuHotspotResults & results, IConstWorkUnit * wu, IPropertyT { WorkunitStatsAnalyser analyser; analyser.applyOptions(cfg); - analyser.analyse(wu); + analyser.analyse(wu, nullptr); analyser.findHotspots(cfg->queryProp("@rootScope"), results.totalTime, results.hotspots); results.root.setown(analyser.getRootScope()); diff --git a/common/wuanalysis/anawu.hpp b/common/wuanalysis/anawu.hpp index 8db7d227426..db2f004bb46 100644 --- a/common/wuanalysis/anawu.hpp +++ b/common/wuanalysis/anawu.hpp @@ -26,7 +26,7 @@ #include "anacommon.hpp" -void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, IPropertyTree *options, double costPerMs); +void WUANALYSIS_API analyseWorkunit(IWorkUnit * wu, const char *optGraph, IPropertyTree *options, double costPerMs); void WUANALYSIS_API analyseAndPrintIssues(IConstWorkUnit * wu, double costPerMs, bool updatewu); //--------------------------------------------------------------------------------------------------------------------- diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 8071d605cb4..55ffbca51e4 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -1856,6 +1856,27 @@ void EclAgent::setRetcode(int code) retcode = code; } + +void EclAgent::runWorkunitAnalyser(IWorkUnit * w, const char * optGraph) +{ + if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + { + double costPerMs = calculateThorCost(1, getNodes()); + IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); + analyseWorkunit(w, optGraph, analyzerOptions, costPerMs); + } +} + +static constexpr bool defaultAnalyzeWhenComplete = true; +void EclAgent::runWorkunitAnalyserAfterGraph(const char * graph) +{ + if (!wuRead->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) + { + Owned wu(updateWorkUnit()); + runWorkunitAnalyser(wu, graph); + } +} + void EclAgent::doProcess() { #ifdef _DEBUG @@ -2026,15 +2047,22 @@ void EclAgent::doProcess() break; } - if (w->getState() == WUStateCompleted && getClusterType(clusterType)==ThorLCRCluster) + + if (getClusterType(clusterType)==ThorLCRCluster) { - if (w->getDebugValueBool("analyzeWorkunit", agentTopology->getPropBool("@analyzeWorkunit", true))) + if (w->getDebugValueBool("analyzeWhenComplete", agentTopology->getPropBool("@analyzeWhenComplete", defaultAnalyzeWhenComplete))) { - double costPerMs = calculateThorCost(1, getNodes()); - IPropertyTree *analyzerOptions = agentTopology->queryPropTree("analyzerOptions"); - analyseWorkunit(w.get(), analyzerOptions, costPerMs); + switch (w->getState()) + { + case WUStateFailed: + case WUStateAborted: + case WUStateCompleted: + runWorkunitAnalyser(w, nullptr); + break; + } } } + if(w->queryEventScheduledCount() > 0) switch(w->getState()) { diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp index 10c86cc6ebb..e16a082755b 100644 --- a/ecl/eclagent/eclagent.ipp +++ b/ecl/eclagent/eclagent.ipp @@ -422,6 +422,8 @@ private: EclAgentQueryLibrary * loadEclLibrary(const char * libraryName, unsigned expectedInterfaceHash, const char * embeddedGraphName); virtual bool getWorkunitResultFilename(StringBuffer & diskFilename, const char * wuid, const char * name, int seq); virtual IDebuggableContext *queryDebugContext() const { return debugContext; }; + void runWorkunitAnalyser(IWorkUnit * w, const char * optGraph); + void runWorkunitAnalyserAfterGraph(const char * optGraph); //protected by critical section EclGraph * addGraph(const char * graphName); diff --git a/ecl/eclagent/eclgraph.cpp b/ecl/eclagent/eclgraph.cpp index d4ffdd99c42..95f533d8c5e 100644 --- a/ecl/eclagent/eclgraph.cpp +++ b/ecl/eclagent/eclgraph.cpp @@ -1598,7 +1598,16 @@ void EclAgent::executeGraph(const char * graphName, bool realThor, size32_t pare { if (isStandAloneExe) throw MakeStringException(0, "Cannot execute Thor Graph in standalone mode"); - executeThorGraph(graphName, *wuRead, *agentTopology); + try + { + executeThorGraph(graphName, *wuRead, *agentTopology); + runWorkunitAnalyserAfterGraph(graphName); + } + catch (...) + { + runWorkunitAnalyserAfterGraph(graphName); + throw; + } } else { From 1b8e41a0692212eea19f3d8537db7ba531919001 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 23 May 2024 18:16:46 +0100 Subject: [PATCH 02/18] Split off 9.0.112 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/localroxie.yaml.fixed | 4 ++-- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/roxie.yaml.fixed | 16 ++++++++-------- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 16 files changed, 35 insertions(+), 35 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index cd689f32b18..f510ae7ad10 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.111-closedown0 +version: 9.0.113-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.111-closedown0 +appVersion: 9.0.113-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 34fe972ae12..d9efa42079f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 478354bde26..cbb749ebc7b 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index c73d45c4185..dc7858734d1 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 9e3ee1607ba..6db713f6534 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 067dddd5bdb..0a4e6a6b28a 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index b269a2030f7..c6b4409a292 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index a051e0a480e..6c08b88e79c 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 60668168a9a..2ce4e023070 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 37f2da967e7..3fc9e88cb61 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml.fixed b/helm/hpcc/templates/localroxie.yaml.fixed index 26904a02308..22a619c9ed4 100644 --- a/helm/hpcc/templates/localroxie.yaml.fixed +++ b/helm/hpcc/templates/localroxie.yaml.fixed @@ -74,13 +74,13 @@ spec: accessDali: "yes" accessEsp: "yes" <<<<<<< HEAD - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} ======= - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 00f0b503d46..9c1a21f0f84 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/roxie.yaml.fixed b/helm/hpcc/templates/roxie.yaml.fixed index b1a388d585c..b8af556380a 100644 --- a/helm/hpcc/templates/roxie.yaml.fixed +++ b/helm/hpcc/templates/roxie.yaml.fixed @@ -126,7 +126,7 @@ spec: run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} <<<<<<< HEAD - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -134,7 +134,7 @@ spec: {{ toYaml $toposerver.labels | indent 8 }} {{- end }} ======= - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -193,10 +193,10 @@ metadata: name: {{ $commonCtx.toponame | quote }} labels: <<<<<<< HEAD - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} ======= - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} >>>>>>> origin/candidate-9.6.x spec: @@ -260,7 +260,7 @@ spec: accessDali: "yes" accessEsp: "yes" <<<<<<< HEAD - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -269,7 +269,7 @@ spec: {{ toYaml $roxie.labels | indent 8 }} {{- end }} ======= - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -379,7 +379,7 @@ spec: accessDali: "yes" accessEsp: "yes" <<<<<<< HEAD - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -387,7 +387,7 @@ spec: {{ toYaml $roxie.labels | indent 8 }} {{- end }} ======= - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 0592ec984db..cb8efb2c45b 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 2ec4ae3ac40..3a18d9ae781 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.111-closedown0 + helmVersion: 9.0.113-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 87c5c4c9492..cc126433607 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 111 ) +set ( HPCC_POINT 113 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-05-17T17:04:28Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-05-23T17:16:46Z" ) ### From 8b0cc2c297e34f092bb4e76792f4531db8436a23 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 23 May 2024 18:20:41 +0100 Subject: [PATCH 03/18] Split off 9.2.90 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index f5fd9c97453..a6063fa3afd 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.89-closedown0 +version: 9.2.91-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.89-closedown0 +appVersion: 9.2.91-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index e1cd6f6b0a3..e00705a2b1f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1405,7 +1405,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index f03fc0bf822..44f0c6243f2 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 3a03b162d9b..876ad3508b0 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 8c1e2228cfa..1174af19bd7 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index a43551f19d0..7e9e10a5d76 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -133,7 +133,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 863a8e51cc2..ff50c624e24 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -140,7 +140,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 6db877a2357..8a34344ab9d 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index be1c1e6a044..148e95577c0 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -120,7 +120,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index c4336c49046..159b0f2bb81 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 54816aea59f..5d512b8ec13 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -178,7 +178,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -240,7 +240,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -346,7 +346,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 98dac05b3e1..0aaa81fd5d7 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index b9671d31fac..bbb397994c2 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -145,7 +145,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -210,7 +210,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -341,7 +341,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -404,7 +404,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.89-closedown0 + helmVersion: 9.2.91-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index aabb113477c..164d057070d 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 89 ) +set ( HPCC_POINT 91 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-05-17T17:05:29Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-05-23T17:20:41Z" ) ### From f25c589f459f2e2fb158ee529ea6c20d61862aff Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 23 May 2024 18:21:58 +0100 Subject: [PATCH 04/18] Split off 9.4.64 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 4 ++-- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index dc6203253eb..bdc71f2b09b 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.4.63-closedown0 +version: 9.4.65-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.4.63-closedown0 +appVersion: 9.4.65-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 3c1663cd1a5..51c3f423080 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1517,7 +1517,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 7f63efcbc85..9ca7bc5e143 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -51,7 +51,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 annotations: checksum/config: {{ $configSHA }} {{- include "hpcc.generateAnnotations" $commonCtx | indent 8 }} diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 5ec9eba13b5..fc36caa774d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -84,7 +84,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 252fa24c8f8..14780166fdd 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -57,7 +57,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 23e52ebfc90..49b761982c8 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -60,7 +60,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -137,7 +137,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index fab020cfea9..e8dc0a634b2 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -143,7 +143,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 5d95808fe9f..3f2c1d9c6d6 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -65,7 +65,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index 7fff8f303e7..a661da1969b 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -122,7 +122,7 @@ spec: accessSasha: "yes" {{- end }} app: {{ $application }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index a76b12bc924..e844550ea24 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -73,7 +73,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index ced46c7f940..216a736a819 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -125,7 +125,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -182,7 +182,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -244,7 +244,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -352,7 +352,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index ac2d7cd4abf..95698f31849 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -53,7 +53,7 @@ spec: server: {{ $serviceName | quote }} app: sasha accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 4d64902b457..5806bfcd1fb 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -86,7 +86,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -151,7 +151,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -218,7 +218,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -351,7 +351,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -416,7 +416,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.4.63-closedown0 + helmVersion: 9.4.65-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index eb6a4cd4248..aa64c86cdaa 100644 --- a/version.cmake +++ b/version.cmake @@ -5,8 +5,8 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 4 ) -set ( HPCC_POINT 63 ) +set ( HPCC_POINT 65 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) -set ( HPCC_TAG_TIMESTAMP "2024-05-17T17:06:31Z" ) +set ( HPCC_TAG_TIMESTAMP "2024-05-23T17:21:58Z" ) ### From 99b754047f6b380bc0d7ffcdde301661c49cdc44 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 24 May 2024 10:28:24 -0400 Subject: [PATCH 05/18] HPCC-31937 ECL Watch v9 fix DFU WUs list % Complete column fixes "% Complete" column in the DFU Workunits list page, which was blank Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/DFUWorkunits.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/DFUWorkunits.tsx b/esp/src/src-react/components/DFUWorkunits.tsx index e0032f2a383..39170270844 100644 --- a/esp/src/src-react/components/DFUWorkunits.tsx +++ b/esp/src/src-react/components/DFUWorkunits.tsx @@ -133,7 +133,7 @@ export const DFUWorkunits: React.FunctionComponent = ({ JobName: { label: nlsHPCC.JobName, width: 220 }, ClusterName: { label: nlsHPCC.Cluster, width: 70 }, StateMessage: { label: nlsHPCC.State, width: 70 }, - PCTDone: { + PercentDone: { label: nlsHPCC.PctComplete, width: 80, sortable: true, }, TimeStarted: { label: nlsHPCC.TimeStarted, width: 100, sortable: true }, From d0b00cd7c77a06c6cc9257631d3884b7193d79ab Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 24 May 2024 15:51:49 +0100 Subject: [PATCH 06/18] HPCC-31927 Legacy graph display regression Generated string[] requests were nested 1 to many redirections. Signed-off-by: Gordon Smith --- esp/src/package-lock.json | 220 ++++++++++---------- esp/src/package.json | 26 +-- esp/src/src-react/components/ECLArchive.tsx | 4 +- esp/src/src-react/components/Metrics.tsx | 6 +- esp/src/src-react/hooks/metrics.ts | 4 +- esp/src/src/ECLArchiveWidget.ts | 4 +- esp/src/src/ESPWorkunit.ts | 4 +- esp/src/src/Timings.ts | 12 +- 8 files changed, 140 insertions(+), 140 deletions(-) diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index 84c45e73947..bc4b85a6248 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -15,20 +15,20 @@ "@fluentui/react-hooks": "8.7.0", "@fluentui/react-icons-mdl2": "1.3.59", "@fluentui/react-migration-v8-v9": "9.6.3", - "@hpcc-js/chart": "2.83.2", - "@hpcc-js/codemirror": "2.61.3", - "@hpcc-js/common": "2.71.16", - "@hpcc-js/comms": "2.92.0", + "@hpcc-js/chart": "2.83.3", + "@hpcc-js/codemirror": "2.61.4", + "@hpcc-js/common": "2.71.17", + "@hpcc-js/comms": "2.92.1", "@hpcc-js/dataflow": "8.1.6", - "@hpcc-js/eclwatch": "2.74.2", - "@hpcc-js/graph": "2.85.14", - "@hpcc-js/html": "2.42.19", - "@hpcc-js/layout": "2.49.21", - "@hpcc-js/map": "2.77.20", - "@hpcc-js/other": "2.15.21", - "@hpcc-js/phosphor": "2.18.7", - "@hpcc-js/react": "2.53.15", - "@hpcc-js/tree": "2.40.16", + "@hpcc-js/eclwatch": "2.74.3", + "@hpcc-js/graph": "2.85.15", + "@hpcc-js/html": "2.42.20", + "@hpcc-js/layout": "2.49.22", + "@hpcc-js/map": "2.77.21", + "@hpcc-js/other": "2.15.22", + "@hpcc-js/phosphor": "2.18.8", + "@hpcc-js/react": "2.53.16", + "@hpcc-js/tree": "2.40.17", "@hpcc-js/util": "2.51.0", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", @@ -1806,35 +1806,35 @@ } }, "node_modules/@hpcc-js/api": { - "version": "2.12.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.16.tgz", - "integrity": "sha512-lmRvwoAHWcrTSKEbe/SR0Y61p4j/+VRnu7CViBe5wQ2nCrK9yFTLZzDlmHhvMIDyVJj4khI4/4ZZCUI13zfuzA==", + "version": "2.12.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.17.tgz", + "integrity": "sha512-NuqjPdxnfbpFQ0e2c0ZBC/hYItPDOTiysO+xfR03SSTqZqxdcME1EEYHI6/wIAb3B+rQ3gk3Xss7EvKq6t+puw==", "dependencies": { - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/chart": { - "version": "2.83.2", - "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.83.2.tgz", - "integrity": "sha512-PePaV/68if4dp+iBDpmBwRbTfOsKht8DkZ7B8NafKfGxL1+khCXf4p4Q/Dlgqig1OfVt35z1eR4lbEUrARhVJw==", + "version": "2.83.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.83.3.tgz", + "integrity": "sha512-CKTnjQ4PhK/g7o/ZvNA3o+AkCLIu/ttkkqDyPmiFQU9an7qXfAVyJyPk9e2/QCeRaTQV1YmMUAj94iZvK9E0Gg==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/codemirror": { - "version": "2.61.3", - "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.61.3.tgz", - "integrity": "sha512-GKLuro8GiMUKu9sCoIsyIXMRbTogG3xGb/yvAuoAKcJQRyJsVamLB7ohyjZkcgY7/JWa3+UBUcABOxiWqG8M/Q==", + "version": "2.61.4", + "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.61.4.tgz", + "integrity": "sha512-rscy1L5EcRhRtldjjwdurxC8RLWW8KY+B8EYj/XXH25blpvlt3P05Bdd6kotBIG18sV33sezaydhM7dqs+iltg==", "dependencies": { - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/common": { - "version": "2.71.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.16.tgz", - "integrity": "sha512-hz5i9zUXBJrXW5tl30XwgFXwJ2nipzLD9pXQrg1Rw8zfXkQ1Xax22RvGZdASAGPsmHxefyWTK7fpcJd+ipOGOg==", + "version": "2.71.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.17.tgz", + "integrity": "sha512-Fyo7U/1hFgV7ZEkiOSj92UYEEtj0T73KV7w09yyD36paSS0wFLTNaFoiFJRVRgmQF8P/LVBgC1lrB7LnYdfFNA==", "dependencies": { "@hpcc-js/util": "^2.51.0", "@types/d3-array": "1.2.12", @@ -1855,9 +1855,9 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.92.0", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.92.0.tgz", - "integrity": "sha512-hGWFUIywb/DHR/yk43C911JY9mwNrion/wt71adfjbckjQJ267GjPkw4tyz8K1YGt5NgCCW+njnR6lAkpjYGfw==", + "version": "2.92.1", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.92.1.tgz", + "integrity": "sha512-nx4JJUSpU1m/Yd53PsbEL26DQFx+3UF0Pelk5O0BY2eFGpPVaQ9jHeWvuMJCHHpHcbxUtWN5nlwZlZgE9CFptA==", "dependencies": { "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/util": "^2.51.0", @@ -1906,11 +1906,11 @@ } }, "node_modules/@hpcc-js/dgrid": { - "version": "2.32.19", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.19.tgz", - "integrity": "sha512-nFKWjepBJIceN2sTMk8N283OFvU5zwfFAeGqBnT3iQRO2vQRaJzZt4G+9xtgVPbnyWuGiqHhIxYoGJLUOMpLbQ==", + "version": "2.32.20", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.20.tgz", + "integrity": "sha512-hCD6nIfWT1RWHBTIAqMs2uJUQSE7hBKLqa6pLFRmlByL7Egev+m5ErR4+aMn9NjqhjE0/HqpwwYnTNoMoVw83w==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/ddl-shim": "^2.20.6", "@hpcc-js/dgrid-shim": "^2.24.10", "@hpcc-js/util": "^2.51.0" @@ -1922,63 +1922,63 @@ "integrity": "sha512-4PD4GvKn2/HQvgzeP+Gd0Halj4KySk0QW1C7dqfyNWV8AUaseT9SSUvyu2ftGPUrzq65sJ0fSaq4zh3Js9dbaQ==" }, "node_modules/@hpcc-js/dgrid2": { - "version": "2.3.18", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.18.tgz", - "integrity": "sha512-7OtEREk9xJYfjDGGP9abSvQWQCDTgwYIx+OfXqFArb031FhTC1rWrkc5svySRb/0VVVAvsQFkHK435GaNy6PHQ==", + "version": "2.3.19", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.19.tgz", + "integrity": "sha512-K0GCX2GR+sayN1glQAEmqdFt+ZcvF3ZU3xwO/LpcA6HGqnN3ena+CeCEE1ufqfLICwPu5P/X+kMd+GJoDY51+w==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/eclwatch": { - "version": "2.74.2", - "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.74.2.tgz", - "integrity": "sha512-FY5CQ/Pezq5enRZtVXzmxV2utv+Fiq7Gn7guMz2IhYWmenNDgclIrfKHeXL8nISJsPNl/VJOHCwyxBWuhuGBdw==", - "dependencies": { - "@hpcc-js/codemirror": "^2.61.3", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/comms": "^2.92.0", - "@hpcc-js/dgrid": "^2.32.19", - "@hpcc-js/graph": "^2.85.14", - "@hpcc-js/layout": "^2.49.21", - "@hpcc-js/phosphor": "^2.18.7", - "@hpcc-js/timeline": "^2.51.24", - "@hpcc-js/tree": "^2.40.16", + "version": "2.74.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.74.3.tgz", + "integrity": "sha512-tsJfXAbREXNXAzui8Mc7Vb9J2xmc1A40I2+pTTOFnVeHPv8bzDvc5sGQXgRrkqqOkeMwzGsnlpbVmC7zTZ33UA==", + "dependencies": { + "@hpcc-js/codemirror": "^2.61.4", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/comms": "^2.92.1", + "@hpcc-js/dgrid": "^2.32.20", + "@hpcc-js/graph": "^2.85.15", + "@hpcc-js/layout": "^2.49.22", + "@hpcc-js/phosphor": "^2.18.8", + "@hpcc-js/timeline": "^2.51.25", + "@hpcc-js/tree": "^2.40.17", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/graph": { - "version": "2.85.14", - "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.14.tgz", - "integrity": "sha512-grofTqK944A8b/LgigDJHBuM9R9+JIDYfqA5wBssbvty3MtLAuN2seoGFn+I7UEojrCggQllKSxNyi/OXoJrCQ==", - "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/html": "^2.42.19", - "@hpcc-js/react": "^2.53.15", + "version": "2.85.15", + "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.15.tgz", + "integrity": "sha512-1LGhS4tywbCPs6b0XObLRiuf3fU16QysTrIA/7F6FUB7w5ay6oiR8tzGTP87SNHZ7fvZ36TAXOyKi/jHJbaf1A==", + "dependencies": { + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/html": "^2.42.20", + "@hpcc-js/react": "^2.53.16", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/html": { - "version": "2.42.19", - "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.19.tgz", - "integrity": "sha512-qocVJXQvwUVaHVXQvn8gIZCXfNHiQOVuMai5wyegYH9KgWJUX3MjUNGHCEYpMsBkk6LBX+D+3myC+VFGlLSgjg==", + "version": "2.42.20", + "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.20.tgz", + "integrity": "sha512-LozHVD0THMJ1IUjbTsmzskoWYobc5siv1S4rgl6sAy1R8etnTgWkpDMgmIFLNS97A3XizpCebJwBfeEN6KsDpg==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/layout": { - "version": "2.49.21", - "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.21.tgz", - "integrity": "sha512-gZSqCBrLDriWW9mhw1bqUL/dzNCsf372CnPjsMLEuMdln7TOOTQm5L0BCjMPVf5kMgyZeT2xRZGr7YURPJIw+g==", + "version": "2.49.22", + "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.22.tgz", + "integrity": "sha512-iNCUUpsA3y6bRQC087Enix3d5vpXoQKxr8FoYVkJFji/UooWU2zJelRPwx7Ky8e1t6Jz2uYxqWI30J5wy8h0HA==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/chart": "^2.83.2", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/dgrid2": "^2.3.18" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/chart": "^2.83.3", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/dgrid2": "^2.3.19" } }, "node_modules/@hpcc-js/leaflet-shim": { @@ -1991,36 +1991,36 @@ } }, "node_modules/@hpcc-js/map": { - "version": "2.77.20", - "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.20.tgz", - "integrity": "sha512-smA6i2viO/DsEaNGIbRKLxHTLWEO8qd7nBgtEFOYugyiiIeyZaBtHVEeTEpOMJAv672L+SxbxAteSbfbdTdd/w==", - "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/graph": "^2.85.14", - "@hpcc-js/layout": "^2.49.21", + "version": "2.77.21", + "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.21.tgz", + "integrity": "sha512-LJHDvpvpllYlapQ9xzqw46oMCyn3WfLNIEeBadrtIoxr7xLFrS2dEXkgydv2BwbRIZNHgjXZPqKfbsWBHqph5w==", + "dependencies": { + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/graph": "^2.85.15", + "@hpcc-js/layout": "^2.49.22", "@hpcc-js/leaflet-shim": "^2.3.5", - "@hpcc-js/other": "^2.15.21", + "@hpcc-js/other": "^2.15.22", "@hpcc-js/util": "^2.51.0" } }, "node_modules/@hpcc-js/other": { - "version": "2.15.21", - "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.21.tgz", - "integrity": "sha512-QUIlQv7nP9+fKNdE8458pqy/cYrEZnVYXBne8uSu2h5q64VSSreCZmO7XY/Rjxf822RigYlZ+fC8K2fZiDOULw==", + "version": "2.15.22", + "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.22.tgz", + "integrity": "sha512-r1dv7Fswrf9SBbmr437k+CpEvzZJJjpGSUCrF2JrMztHpsFZed4qlo9agIVp5RksXxQf7SPMI9IXWthYIUG5LQ==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/layout": "^2.49.21" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/layout": "^2.49.22" } }, "node_modules/@hpcc-js/phosphor": { - "version": "2.18.7", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.18.7.tgz", - "integrity": "sha512-iSQX6vIpawQPbDVhc/CbH8Z4ysSzb+uFjeasd1zIfE77Km5ImH9IiI9OZMOoFYi1zCTCfce/Y7NLC8ADOgg2XQ==", + "version": "2.18.8", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.18.8.tgz", + "integrity": "sha512-/D7lXuuPoeUuCmaBv/JMjmThKu3zl0j2/m7LxTJ9ltFmu7KkMNQK6EYLc0+VRVWrxa9P8hXMlLCXN0YlXtKTww==", "dependencies": { - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/other": "^2.15.21", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/other": "^2.15.22", "@hpcc-js/phosphor-shim": "^2.14.6", "@hpcc-js/util": "^2.51.0" } @@ -2045,34 +2045,34 @@ } }, "node_modules/@hpcc-js/react": { - "version": "2.53.15", - "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.15.tgz", - "integrity": "sha512-X8e1lIk4oRXFNTFxrcZ1YbJDi6t7IU431Lzq0nTIFJqWIDRZgjJ3gKxSGcjtjNYzhUqq4A9KfcdvKNE+wHrdjw==", + "version": "2.53.16", + "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.16.tgz", + "integrity": "sha512-pJ0/hE2MOCnaWFWBRyx1TEXV6x0bwPkoEKmwmGb72pCJQoVPaSKQ8bEi+UOHX+xxw0TF5x8u7fJHUjxvdcYMmQ==", "dependencies": { - "@hpcc-js/common": "^2.71.16", + "@hpcc-js/common": "^2.71.17", "@hpcc-js/preact-shim": "^2.16.10" } }, "node_modules/@hpcc-js/timeline": { - "version": "2.51.24", - "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.24.tgz", - "integrity": "sha512-QNgXhJ6/hQHfP2Lge2zL1X5ERI813KKpFN+DNFqufhWoZIT/7x3kr1If8r1mC74hYt4xqkFAdoveEepFT+lYhQ==", + "version": "2.51.25", + "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.25.tgz", + "integrity": "sha512-SS/67TomcrJaaMrtWYs9fk/TK0hr/ve/oBVux5Ibnph826xBB3Z5wt3tzfWYgzlvcsdGPFi6TF4FKFtYOTX9LA==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/chart": "^2.83.2", - "@hpcc-js/common": "^2.71.16", - "@hpcc-js/html": "^2.42.19", - "@hpcc-js/layout": "^2.49.21", - "@hpcc-js/react": "^2.53.15" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/chart": "^2.83.3", + "@hpcc-js/common": "^2.71.17", + "@hpcc-js/html": "^2.42.20", + "@hpcc-js/layout": "^2.49.22", + "@hpcc-js/react": "^2.53.16" } }, "node_modules/@hpcc-js/tree": { - "version": "2.40.16", - "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.16.tgz", - "integrity": "sha512-UCFA3ky9aB0XqrN4PyNmwkY3zl3VSc4araEfHpjtOcT7r7pUVJNEG+KjYPkCTUvvKYoPIuE2FBGtr6ec0bM5Aw==", + "version": "2.40.17", + "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.17.tgz", + "integrity": "sha512-Z8uTo6281tcTMLUmNynYJEn5MiS5qkm76FcalgMJKaOduoUXL1YezDGFqo23y61JL6GpHdVRz/go9B5uI+Sapg==", "dependencies": { - "@hpcc-js/api": "^2.12.16", - "@hpcc-js/common": "^2.71.16" + "@hpcc-js/api": "^2.12.17", + "@hpcc-js/common": "^2.71.17" } }, "node_modules/@hpcc-js/util": { diff --git a/esp/src/package.json b/esp/src/package.json index a7685259e97..f823572534d 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -41,20 +41,20 @@ "@fluentui/react-hooks": "8.7.0", "@fluentui/react-icons-mdl2": "1.3.59", "@fluentui/react-migration-v8-v9": "9.6.3", - "@hpcc-js/chart": "2.83.2", - "@hpcc-js/codemirror": "2.61.3", - "@hpcc-js/common": "2.71.16", - "@hpcc-js/comms": "2.92.0", + "@hpcc-js/chart": "2.83.3", + "@hpcc-js/codemirror": "2.61.4", + "@hpcc-js/common": "2.71.17", + "@hpcc-js/comms": "2.92.1", "@hpcc-js/dataflow": "8.1.6", - "@hpcc-js/eclwatch": "2.74.2", - "@hpcc-js/graph": "2.85.14", - "@hpcc-js/html": "2.42.19", - "@hpcc-js/layout": "2.49.21", - "@hpcc-js/map": "2.77.20", - "@hpcc-js/other": "2.15.21", - "@hpcc-js/phosphor": "2.18.7", - "@hpcc-js/react": "2.53.15", - "@hpcc-js/tree": "2.40.16", + "@hpcc-js/eclwatch": "2.74.3", + "@hpcc-js/graph": "2.85.15", + "@hpcc-js/html": "2.42.20", + "@hpcc-js/layout": "2.49.22", + "@hpcc-js/map": "2.77.21", + "@hpcc-js/other": "2.15.22", + "@hpcc-js/phosphor": "2.18.8", + "@hpcc-js/react": "2.53.16", + "@hpcc-js/tree": "2.40.17", "@hpcc-js/util": "2.51.0", "@kubernetes/client-node": "0.20.0", "clipboard": "2.0.11", diff --git a/esp/src/src-react/components/ECLArchive.tsx b/esp/src/src-react/components/ECLArchive.tsx index 7f43a368726..8d63d3ae8f5 100644 --- a/esp/src/src-react/components/ECLArchive.tsx +++ b/esp/src/src-react/components/ECLArchive.tsx @@ -17,12 +17,12 @@ const logger = scopedLogger("src-react/components/ECLArchive.tsx"); const scopeFilterDefault: Partial = { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }; const nestedFilterDefault: WsWorkunits.NestedFilter = { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }; interface ECLArchiveProps { diff --git a/esp/src/src-react/components/Metrics.tsx b/esp/src/src-react/components/Metrics.tsx index a4c43170dac..5373bb6ab59 100644 --- a/esp/src/src-react/components/Metrics.tsx +++ b/esp/src/src-react/components/Metrics.tsx @@ -198,17 +198,17 @@ export const Metrics: React.FunctionComponent = ({ .request({ ScopeFilter: { MaxDepth: 3, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: ["WhenStarted", "TimeElapsed", "TimeLocalExecute"] } + Properties: ["WhenStarted", "TimeElapsed", "TimeLocalExecute"] }, ScopeOptions: { IncludeId: true, diff --git a/esp/src/src-react/hooks/metrics.ts b/esp/src/src-react/hooks/metrics.ts index efdfa3da2ea..d377997ac92 100644 --- a/esp/src/src-react/hooks/metrics.ts +++ b/esp/src/src-react/hooks/metrics.ts @@ -104,12 +104,12 @@ export enum FetchStatus { const scopeFilterDefault: Partial = { MaxDepth: 999999, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }; const nestedFilterDefault: WsWorkunits.NestedFilter = { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }; export function useWorkunitMetrics( diff --git a/esp/src/src/ECLArchiveWidget.ts b/esp/src/src/ECLArchiveWidget.ts index 2f5efaa752a..8076a41433e 100644 --- a/esp/src/src/ECLArchiveWidget.ts +++ b/esp/src/src/ECLArchiveWidget.ts @@ -171,7 +171,7 @@ export class ECLArchiveWidget { const scopesOptions: RecursivePartial = { ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }, ScopeOptions: { IncludeMatchedScopesInResults: true, @@ -189,7 +189,7 @@ export class ECLArchiveWidget { }, NestedFilter: { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }, PropertiesToReturn: { AllStatistics: true, diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 428eb156107..4278d060d26 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -917,7 +917,7 @@ const Workunit = declare([ESPUtil.Singleton], { // jshint ignore:line return (this._hpccWU as HPCCWorkunit).fetchDetails({ ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: ["graph"] } + ScopeTypes: ["graph"] }, ScopeOptions: { IncludeMatchedScopesInResults: true, @@ -935,7 +935,7 @@ const Workunit = declare([ESPUtil.Singleton], { // jshint ignore:line }, NestedFilter: { Depth: 999999, - ScopeTypes: { ScopeType: ["activity"] } + ScopeTypes: ["activity"] }, PropertiesToReturn: { AllStatistics: false, diff --git a/esp/src/src/Timings.ts b/esp/src/src/Timings.ts index cea27ccbc3d..c7db66385e5 100644 --- a/esp/src/src/Timings.ts +++ b/esp/src/src/Timings.ts @@ -69,17 +69,17 @@ export class Timings { .request({ ScopeFilter: { MaxDepth: 3, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: ["WhenStarted", "TimeElapsed"] } + Properties: ["WhenStarted", "TimeElapsed"] }, ScopeOptions: { IncludeId: true, @@ -210,17 +210,17 @@ export class Timings { this.fetchDetailsNormalizedPromise = Promise.all([this.wu.fetchDetailsMeta(), this.wu.fetchDetailsRaw({ ScopeFilter: { MaxDepth: 999999, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, NestedFilter: { Depth: 0, - ScopeTypes: { ScopeType: [] } + ScopeTypes: [] }, PropertiesToReturn: { AllProperties: false, AllStatistics: true, AllHints: false, - Properties: { Property: [] } + Properties: [] }, ScopeOptions: { IncludeId: true, From 927372ebebbfe76c4f639949696f05596bf01749 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 26 Feb 2024 16:37:30 +0000 Subject: [PATCH 07/18] HPCC-31353 Add override attributes Signed-off-by: Gavin Halliday --- roxie/ccd/ccdserver.cpp | 82 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index 2cf5cbd6d40..ff269d51796 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -176,55 +176,55 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface // void set(IRoxieAgentContext * _ctx) { ctx = _ctx; } - virtual ICodeContext *queryCodeContext() + virtual ICodeContext *queryCodeContext() override { return ctx->queryCodeContext(); } - virtual void checkAbort() + virtual void checkAbort() override { ctx->checkAbort(); } - virtual unsigned checkInterval() const + virtual unsigned checkInterval() const override { return ctx->checkInterval(); } - virtual void notifyAbort(IException *E) + virtual void notifyAbort(IException *E) override { ctx->notifyAbort(E); } - virtual void notifyException(IException *E) + virtual void notifyException(IException *E) override { ctx->notifyException(E); } - virtual void throwPendingException() + virtual void throwPendingException() override { ctx->throwPendingException(); } - virtual IActivityGraph * queryChildGraph(unsigned id) + virtual IActivityGraph * queryChildGraph(unsigned id) override { return ctx->queryChildGraph(id); } - virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph) + virtual void noteChildGraph(unsigned id, IActivityGraph *childGraph) override { ctx->noteChildGraph(id, childGraph) ; } - virtual IRowManager &queryRowManager() + virtual IRowManager &queryRowManager() override { return ctx->queryRowManager(); } - virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const + virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const override { ctx->noteStatistic(kind, value); } - virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const + virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const override { ctx->setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(const CRuntimeStatisticCollection &from) const override { ctx->mergeStats(from); } - virtual StringBuffer &getStats(StringBuffer &ret) const + virtual StringBuffer &getStats(StringBuffer &ret) const override { return ctx->getStats(ret); } @@ -236,7 +236,7 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->recordStatistics(progress); } - virtual bool collectingDetailedStatistics() const + virtual bool collectingDetailedStatistics() const override { return ctx->collectingDetailedStatistics(); } @@ -248,15 +248,15 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->CTXLOGa(category, cat, job, code, prefix, text); } - virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) + virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const override __attribute__((format(printf,5,0))) { ctx->logOperatorExceptionVA(E, file, line, format, args); } - virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const __attribute__((format(printf,6,0))) + virtual void CTXLOGaeva(IException *E, const char *file, unsigned line, const char *prefix, const char *format, va_list args) const override __attribute__((format(printf,6,0))) { ctx->CTXLOGaeva(E, file, line, prefix, format, args); } - virtual void CTXLOGl(LogItem *log) const + virtual void CTXLOGl(LogItem *log) const override { ctx->CTXLOGl(log); } @@ -264,15 +264,15 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->getLogPrefix(ret); } - virtual unsigned queryTraceLevel() const + virtual unsigned queryTraceLevel() const override { return ctx->queryTraceLevel(); } - virtual bool isIntercepted() const + virtual bool isIntercepted() const override { return ctx->isIntercepted(); } - virtual bool isBlind() const + virtual bool isBlind() const override { return ctx->isBlind(); } @@ -300,7 +300,7 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->setSpanAttribute(name, value); } - virtual const char *queryGlobalId() const + virtual const char *queryGlobalId() const override { return ctx->queryGlobalId(); } @@ -308,83 +308,83 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { return ctx->queryCallerId(); } - virtual const char *queryLocalId() const + virtual const char *queryLocalId() const override { return ctx->queryLocalId(); } - virtual const QueryOptions &queryOptions() const + virtual const QueryOptions &queryOptions() const override { return ctx->queryOptions(); } - virtual void addAgentsReplyLen(unsigned len, unsigned duplicates, unsigned resends) + virtual void addAgentsReplyLen(unsigned len, unsigned duplicates, unsigned resends) override { ctx->addAgentsReplyLen(len, duplicates, resends); } - virtual const char *queryAuthToken() + virtual const char *queryAuthToken() override { return ctx->queryAuthToken(); } - virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser) + virtual const IResolvedFile *resolveLFN(const char *filename, bool isOpt, bool isPrivilegedUser) override { return ctx->resolveLFN(filename, isOpt, isPrivilegedUser); } - virtual IRoxieWriteHandler *createWriteHandler(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser) + virtual IRoxieWriteHandler *createWriteHandler(const char *filename, bool overwrite, bool extend, const StringArray &clusters, bool isPrivilegedUser) override { return ctx->createWriteHandler(filename, overwrite, extend, clusters, isPrivilegedUser); } - virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser) + virtual void onFileCallback(const RoxiePacketHeader &header, const char *lfn, bool isOpt, bool isLocal, bool isPrivilegedUser) override { ctx->onFileCallback(header, lfn, isOpt, isLocal, isPrivilegedUser); } - virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity) + virtual IActivityGraph *getLibraryGraph(const LibraryCallFactoryExtra &extra, IRoxieServerActivity *parentActivity) override { return ctx->getLibraryGraph(extra, parentActivity); } - virtual IProbeManager *queryProbeManager() const + virtual IProbeManager *queryProbeManager() const override { return ctx->queryProbeManager(); } - virtual IDebuggableContext *queryDebugContext() const + virtual IDebuggableContext *queryDebugContext() const override { return ctx->queryDebugContext(); } - virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) + virtual void printResults(IXmlWriter *output, const char *name, unsigned sequence) override { ctx->printResults(output, name, sequence); } - virtual void setWUState(WUState state) + virtual void setWUState(WUState state) override { ctx->setWUState(state); } - virtual bool checkWuAborted() + virtual bool checkWuAborted() override { return ctx->checkWuAborted(); } - virtual IWorkUnit *updateWorkUnit() const + virtual IWorkUnit *updateWorkUnit() const override { return ctx->updateWorkUnit(); } - virtual IConstWorkUnit *queryWorkUnit() const + virtual IConstWorkUnit *queryWorkUnit() const override { return ctx->queryWorkUnit(); } - virtual IRoxieServerContext *queryServerContext() + virtual IRoxieServerContext *queryServerContext() override { return ctx->queryServerContext(); } - virtual IWorkUnitRowReader *getWorkunitRowReader(const char *wuid, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) + virtual IWorkUnitRowReader *getWorkunitRowReader(const char *wuid, const char * name, unsigned sequence, IXmlToRowTransformer * xmlTransformer, IEngineRowAllocator *rowAllocator, bool isGrouped) override { return ctx->getWorkunitRowReader(wuid, name, sequence, xmlTransformer, rowAllocator, isGrouped); } - virtual IEngineRowAllocator *getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, roxiemem::RoxieHeapFlags flags) const + virtual IEngineRowAllocator *getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, roxiemem::RoxieHeapFlags flags) const override { return ctx->getRowAllocatorEx(meta, activityId, flags); } - virtual void noteLibrary(IQueryFactory *library) + virtual void noteLibrary(IQueryFactory *library) override { ctx->noteLibrary(library); } - virtual const CRuntimeStatisticCollection & queryStats() const + virtual const CRuntimeStatisticCollection & queryStats() const override { return ctx->queryStats(); } From 1cb1f1e3819d26198382bd83cbcd4b031365a59a Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Mon, 26 Feb 2024 16:49:28 +0000 Subject: [PATCH 08/18] HPCC-31353 Report the slowest 5 activies in the roxie complete line Signed-off-by: Gavin Halliday --- common/thorhelper/thorcommon.hpp | 2 +- roxie/ccd/ccd.hpp | 23 +--- roxie/ccd/ccdcontext.cpp | 8 +- roxie/ccd/ccdlistener.cpp | 78 ++++++++++++ roxie/ccd/ccdserver.cpp | 14 +-- roxie/ccd/ccdserver.hpp | 2 +- system/jlib/jlog.cpp | 2 +- system/jlib/jlog.hpp | 2 +- testing/unittests/jlibtests.cpp | 112 +++++++++++++++++- .../activities/keyedjoin/thkeyedjoinslave.cpp | 2 +- 10 files changed, 210 insertions(+), 35 deletions(-) diff --git a/common/thorhelper/thorcommon.hpp b/common/thorhelper/thorcommon.hpp index 78123e6e7c7..c8ceebe960a 100644 --- a/common/thorhelper/thorcommon.hpp +++ b/common/thorhelper/thorcommon.hpp @@ -712,7 +712,7 @@ class CStatsContextLogger : public CSimpleInterfaceOf { stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const override + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { stats.merge(from); } diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index b2e447dc5ba..ddf70217a05 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -592,6 +592,9 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface bool blind; mutable bool aborted; mutable CIArrayOf log; + static constexpr const unsigned MaxSlowActivities = 5; + mutable unsigned slowestActivityIds[MaxSlowActivities] = {}; + mutable stat_type slowestActivityTimes[MaxSlowActivities] = {}; private: Owned activeSpan = getNullSpan(); ContextLogger(const ContextLogger &); // Disable copy constructor @@ -693,11 +696,7 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface ctxTraceLevel = _level; } - StringBuffer &getStats(StringBuffer &s) const - { - CriticalBlock block(statsCrit); - return stats.toStr(s); - } + StringBuffer &getStats(StringBuffer &s) const; virtual bool isIntercepted() const { @@ -721,18 +720,8 @@ class ContextLogger : implements IRoxieContextLogger, public CInterface stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const - { - if (from.isThreadSafeMergeSource()) - { - stats.merge(from); - } - else - { - CriticalBlock block(statsCrit); - stats.merge(from); - } - } + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const; + virtual void gatherStats(CRuntimeStatisticCollection & merged) const override { merged.merge(stats); diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 4849a1e1691..0c6037eb40e 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -1313,9 +1313,9 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext logctx.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const override + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { - logctx.mergeStats(from); + logctx.mergeStats(activityId, from); } virtual void gatherStats(CRuntimeStatisticCollection & merged) const override @@ -2379,7 +2379,7 @@ class CAgentContext : public CRoxieContextBase { // NOTE: This is needed to ensure that owned activities are destroyed BEFORE I am, // to avoid pure virtual calls when they come to call noteProcessed() - logctx.mergeStats(globalStats); + logctx.mergeStats(0, globalStats); if (factory) factory->mergeStats(logctx); childGraphs.releaseAll(); @@ -2611,7 +2611,7 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon void doPostProcess() { - logctx.mergeStats(globalStats); + logctx.mergeStats(0, globalStats); logctx.setStatistic(StTimeTotalExecute, elapsedTimer.elapsedNs()); if (factory) { diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index 6e319d2ae67..d70dde2fa36 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -945,6 +945,61 @@ extern void updateAffinity(unsigned __int64 affinity) //-------------------------------------------------------------------------------------------------------------------- +StringBuffer & ContextLogger::getStats(StringBuffer &s) const +{ + CriticalBlock block(statsCrit); + stats.toStr(s); + + if (slowestActivityIds[0]) + { + StringBuffer ids; + StringBuffer times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowestActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowestActivityIds[i]); + formatStatistic(times, cycle_to_nanosec(slowestActivityTimes[i]), SMeasureTimeNs); + } + s.appendf(", slowestActivities={ ids=[%s] times=[%s] }", ids.str(), times.str()); + } + return s; +} + + +void ContextLogger::mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const +{ + CLeavableCriticalBlock block(statsCrit, !from.isThreadSafeMergeSource()); + + stats.merge(from); + + //Record the times of the slowest N activities + if (activityId) + { + stat_type localTime = from.getStatisticValue(StCycleLocalExecuteCycles); + if (localTime > slowestActivityTimes[MaxSlowActivities-1]) + { + unsigned pos = MaxSlowActivities-1; + while (pos > 0) + { + if (localTime <= slowestActivityTimes[pos-1]) + break; + slowestActivityIds[pos] = slowestActivityIds[pos-1]; + slowestActivityTimes[pos] = slowestActivityTimes[pos-1]; + pos--; + } + slowestActivityIds[pos] = activityId; + slowestActivityTimes[pos] = localTime; + } + } +} + void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned memused, unsigned agentsDuplicates, unsigned agentsResends) { if (activeSpan->isRecording()) @@ -957,6 +1012,29 @@ void ContextLogger::exportStatsToSpan(bool failed, stat_type elapsedNs, unsigned StringBuffer prefix(""); stats.exportToSpan(activeSpan, prefix); + + if (slowestActivityIds[0]) + { + //Even better if these were exported as arrays - needs extensions to our api + //Not commoned up with the code above because it is likely to change to arrays in the future. + StringBuffer ids; + StringBuffer times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowestActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowestActivityIds[i]); + times.append(cycle_to_nanosec(slowestActivityTimes[i])); + } + setSpanAttribute("slowest_activities.ids", ids); + setSpanAttribute("slowest_activities.times", times); + } } } diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp index ff269d51796..2b68f0cb3e7 100644 --- a/roxie/ccd/ccdserver.cpp +++ b/roxie/ccd/ccdserver.cpp @@ -220,9 +220,9 @@ class IndirectAgentContext : implements IRoxieAgentContext, public CInterface { ctx->setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const override + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const override { - ctx->mergeStats(from); + ctx->mergeStats(activityId, from); } virtual StringBuffer &getStats(StringBuffer &ret) const override { @@ -681,7 +681,7 @@ class CRoxieServerActivityFactoryBase : public CActivityFactory, implements IRox throwUnexpected(); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(const CRuntimeStatisticCollection &from) const override { CActivityFactory::mergeStats(from); } @@ -1265,7 +1265,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl //Updates the query summary statistics if (ctx) - ctx->queryCodeContext()->queryContextLogger().mergeStats(mergedStats); + ctx->queryCodeContext()->queryContextLogger().mergeStats(activityId, mergedStats); ForEachItemIn(i, childGraphs) childGraphs.item(i).gatherStatistics(statsBuilder); @@ -1287,7 +1287,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { stats.deserializeMerge(buf); } - virtual void mergeStats(const CRuntimeStatisticCollection & childStats) + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection & childStats) { CriticalBlock b(statscrit); stats.merge(childStats); @@ -1383,7 +1383,7 @@ class CRoxieServerActivity : implements CInterfaceOf, impl { stats.setStatistic(kind, value); } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const { stats.merge(from); } @@ -5212,7 +5212,7 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie CRuntimeStatisticCollection childStats(allStatistics); childStats.deserialize(buf); //activity.queryContext()->mergeActivityStats(childStats, graphId, childId); - activity.mergeStats(childStats); + activity.mergeStats(0, childStats); } } ReleaseRoxieRow(logInfo); diff --git a/roxie/ccd/ccdserver.hpp b/roxie/ccd/ccdserver.hpp index 402436abdc2..a1a5c7e452b 100644 --- a/roxie/ccd/ccdserver.hpp +++ b/roxie/ccd/ccdserver.hpp @@ -198,7 +198,7 @@ interface IRoxieServerActivity : extends IActivityBase virtual ThorActivityKind getKind() const = 0; virtual const IRoxieContextLogger &queryLogCtx() const = 0; virtual void mergeStats(MemoryBuffer &stats) = 0; - virtual void mergeStats(const CRuntimeStatisticCollection & childStats) = 0; + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection & childStats) = 0; virtual ISectionTimer * registerTimer(unsigned activityId, const char * name) = 0; virtual IEngineRowAllocator * createRowAllocator(IOutputMetaData * metadata) = 0; virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const = 0; diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 6039e30ba71..714c46251c4 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -2866,7 +2866,7 @@ class DummyLogCtx : implements IContextLogger virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const { } - virtual void mergeStats(const CRuntimeStatisticCollection &from) const + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const { } virtual unsigned queryTraceLevel() const diff --git a/system/jlib/jlog.hpp b/system/jlib/jlog.hpp index 02d5dfff2e6..1084f5f2b06 100644 --- a/system/jlib/jlog.hpp +++ b/system/jlib/jlog.hpp @@ -1264,7 +1264,7 @@ interface jlib_decl IContextLogger : extends IInterface virtual void logOperatorExceptionVA(IException *E, const char *file, unsigned line, const char *format, va_list args) const __attribute__((format(printf,5,0))) = 0; virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const = 0; virtual void setStatistic(StatisticKind kind, unsigned __int64 value) const = 0; - virtual void mergeStats(const CRuntimeStatisticCollection &from) const = 0; + virtual void mergeStats(unsigned activityId, const CRuntimeStatisticCollection &from) const = 0; virtual unsigned queryTraceLevel() const = 0; virtual const char *queryGlobalId() const = 0; diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index 4ea638fad96..289c1e0325f 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -4000,6 +4000,116 @@ CPPUNIT_TEST_SUITE_REGISTRATION( JLibOpensslAESTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibOpensslAESTest, "JLibOpensslAESTest" ); #endif +class TopNStressTest : public CppUnit::TestFixture +{ +public: + CPPUNIT_TEST_SUITE(TopNStressTest); + CPPUNIT_TEST(testTop5a); + CPPUNIT_TEST(testTop5b); + CPPUNIT_TEST_SUITE_END(); + + struct x + { + static constexpr const unsigned MaxSlowActivities = 5; + mutable unsigned slowActivityIds[MaxSlowActivities] = {}; + mutable stat_type slowActivityTimes[MaxSlowActivities] = {}; + unsigned cnt = 0; + + __attribute__((noinline)) + void noteTime(unsigned activityId, stat_type localTime) + { + if (localTime > slowActivityTimes[MaxSlowActivities-1]) + { + unsigned pos = MaxSlowActivities-1; + while (pos > 0) + { + if (localTime <= slowActivityTimes[pos-1]) + break; + slowActivityIds[pos] = slowActivityIds[pos-1]; + slowActivityTimes[pos] = slowActivityTimes[pos-1]; + pos--; + } + slowActivityIds[pos] = activityId; + slowActivityTimes[pos] = localTime; + cnt++; + } + } + __attribute__((noinline)) + void noteTime2(unsigned activityId, stat_type localTime) + { + if (localTime > slowActivityTimes[0]) + { + unsigned pos = 1; + while (pos < MaxSlowActivities) + { + if (localTime <= slowActivityTimes[pos]) + break; + slowActivityIds[pos-1] = slowActivityIds[pos]; + slowActivityTimes[pos-1] = slowActivityTimes[pos]; + pos++; + } + slowActivityIds[pos-1] = activityId; + slowActivityTimes[pos-1] = localTime; + cnt++; + } + } + void report(__uint64 elapsedNs) + { + StringBuffer ids, times; + for (unsigned i=0; i < MaxSlowActivities; i++) + { + if (!slowActivityIds[i]) + break; + + if (i) + { + ids.append(","); + times.append(","); + } + ids.append(slowActivityIds[i]); + formatStatistic(times, cycle_to_nanosec(slowActivityTimes[i]), SMeasureTimeNs); + } + DBGLOG("SlowActivities={ ids=[%s] times=[%s] } in %llu (x%u)", ids.str(), times.str(), elapsedNs, cnt); + + } + }; + + static constexpr const unsigned NumIters = 5000; + + void testTop5a() + { + CCycleTimer timer; + stat_type value = 0; + x tally; + for (unsigned activityId = 1; activityId <= NumIters; activityId++) + { + value = value * 0x100000001b3ULL + 99; + tally.noteTime(activityId, value); + } + + __uint64 elapsedNs = timer.elapsedNs(); + tally.report(elapsedNs); + } + void testTop5b() + { + CCycleTimer timer; + stat_type value = 0; + x tally; + for (unsigned activityId = 1; activityId <= NumIters; activityId++) + { + value = value * 0x100000001b3ULL + 99; + tally.noteTime2(activityId, value); + } + + __uint64 elapsedNs = timer.elapsedNs(); + tally.report(elapsedNs); + } + +}; + +CPPUNIT_TEST_SUITE_REGISTRATION( TopNStressTest ); +CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( TopNStressTest, "TopNStressTest" ); + class JLibSecretsTest : public CppUnit::TestFixture { public: @@ -4325,6 +4435,4 @@ class JLibSecretsTest : public CppUnit::TestFixture CPPUNIT_TEST_SUITE_REGISTRATION( JLibSecretsTest ); CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( JLibSecretsTest, "JLibSecretsTest" ); - - #endif // _USE_CPPUNIT diff --git a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp index 2c5e880d2d8..328428b0520 100644 --- a/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp +++ b/thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp @@ -1768,7 +1768,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor, implem CRuntimeStatisticCollection statsDelta(jhtreeCacheStatistics); statsDelta.deserialize(mb); CStatsContextLogger * contextLogger(contextLoggers[selected]); - contextLogger->mergeStats(statsDelta); + contextLogger->mergeStats(activity.queryActivityId(), statsDelta); if (received == numRows) break; } From 47133ebc9bfb50ba91a9161dc2ec0f9630c4abf3 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 28 May 2024 13:33:49 +0100 Subject: [PATCH 09/18] Add a minimum time to the slowest roxie activies Signed-off-by: Gavin Halliday --- roxie/ccd/ccd.hpp | 2 ++ roxie/ccd/ccdlistener.cpp | 23 +++++++++++++---------- roxie/ccd/ccdmain.cpp | 4 ++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/roxie/ccd/ccd.hpp b/roxie/ccd/ccd.hpp index ddf70217a05..180effdc256 100644 --- a/roxie/ccd/ccd.hpp +++ b/roxie/ccd/ccd.hpp @@ -452,6 +452,8 @@ extern unsigned agentQueryReleaseDelaySeconds; extern unsigned coresPerQuery; extern unsigned cacheReportPeriodSeconds; +extern stat_type minimumInterestingActivityCycles; + extern StringBuffer logDirectory; extern StringBuffer pluginDirectory; diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index d70dde2fa36..00e2e5994fe 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -983,19 +983,22 @@ void ContextLogger::mergeStats(unsigned activityId, const CRuntimeStatisticColle if (activityId) { stat_type localTime = from.getStatisticValue(StCycleLocalExecuteCycles); - if (localTime > slowestActivityTimes[MaxSlowActivities-1]) + if (localTime >= minimumInterestingActivityCycles) { - unsigned pos = MaxSlowActivities-1; - while (pos > 0) + if (localTime > slowestActivityTimes[MaxSlowActivities-1]) { - if (localTime <= slowestActivityTimes[pos-1]) - break; - slowestActivityIds[pos] = slowestActivityIds[pos-1]; - slowestActivityTimes[pos] = slowestActivityTimes[pos-1]; - pos--; + unsigned pos = MaxSlowActivities-1; + while (pos > 0) + { + if (localTime <= slowestActivityTimes[pos-1]) + break; + slowestActivityIds[pos] = slowestActivityIds[pos-1]; + slowestActivityTimes[pos] = slowestActivityTimes[pos-1]; + pos--; + } + slowestActivityIds[pos] = activityId; + slowestActivityTimes[pos] = localTime; } - slowestActivityIds[pos] = activityId; - slowestActivityTimes[pos] = localTime; } } } diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index 9033620344d..2383c0fbee9 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -209,6 +209,7 @@ unsigned __int64 minFreeDiskSpace = 1024 * 0x100000; // default to 1 GB unsigned socketCheckInterval = 5000; unsigned cacheReportPeriodSeconds = 5*60; +stat_type minimumInterestingActivityCycles; StringBuffer logDirectory; StringBuffer pluginDirectory; @@ -1320,6 +1321,9 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) unsigned __int64 affinity = topology->getPropInt64("@affinity", 0); updateAffinity(affinity); + unsigned __int64 minimumInterestingActivityMs = topology->getPropInt64("@minimumInterestingActivityMs", 10); + minimumInterestingActivityCycles = nanosec_to_cycle(minimumInterestingActivityMs * 1'000'000); + minFreeDiskSpace = topology->getPropInt64("@minFreeDiskSpace", (1024 * 0x100000)); // default to 1 GB mtu_size = topology->getPropInt("@mtuPayload", 0); if (mtu_size) From 55ba285bb8035922020aed9e28c2cbaac5a3a8d4 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 28 May 2024 11:13:59 -0400 Subject: [PATCH 10/18] HPCC-31714 ECL Watch v9 Files filter by multiple clusters Changes the "Cluster" field in the Files list's Filter dialog to allow multiple values. Also, changes the AsyncDropdown component to accept an optional value separator property, which defaults to a "|" Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 2 +- esp/src/src-react/components/forms/Fields.tsx | 32 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 01a1e53eca8..14939bd6164 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -33,7 +33,7 @@ const FilterFields: Fields = { "SuperFiles": { type: "checkbox", label: nlsHPCC.SuperFiles }, "Indexes": { type: "checkbox", label: nlsHPCC.Indexes }, "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles, disabled: (params: Fields) => !!params?.SuperFiles?.value || !!params?.LogicalFiles?.value }, - "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster }, + "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster, multiSelect: true, valueSeparator: "," }, "FileSizeFrom": { type: "string", label: nlsHPCC.FromSizes, placeholder: "4096" }, "FileSizeTo": { type: "string", label: nlsHPCC.ToSizes, placeholder: "16777216" }, "FileType": { type: "file-type", label: nlsHPCC.FileType }, diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index 5a4bbc0078f..10775ffc83e 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -82,6 +82,7 @@ interface AsyncDropdownProps { required?: boolean; disabled?: boolean; multiSelect?: boolean; + valueSeparator?: string; errorMessage?: string; onChange?: (event: React.FormEvent, option?: IDropdownOption | IDropdownOption[], index?: number) => void; placeholder?: string; @@ -95,6 +96,7 @@ const AsyncDropdown: React.FunctionComponent = ({ required = false, disabled, multiSelect = false, + valueSeparator = "|", errorMessage, onChange, placeholder, @@ -114,7 +116,7 @@ const AsyncDropdown: React.FunctionComponent = ({ const [selectedItems, setSelectedItems] = React.useState([]); const changeSelectedItems = React.useCallback(() => { - const keys = selectedKey !== "" ? selectedKey.split("|") : []; + const keys = selectedKey !== "" ? selectedKey.split(valueSeparator) : []; let items = [...selectedItems]; if (keys.length === items.length) return; if (selectedKeys !== "" && selOptions.length && selectedKey === "") { @@ -123,16 +125,16 @@ const AsyncDropdown: React.FunctionComponent = ({ } items = keys.map(key => { return { key: key, text: key }; }); if (!items.length) return; - if (items.map(item => item.key).join("|") === selectedKey) { + if (items.map(item => item.key).join(valueSeparator) === selectedKey) { // do nothing, unless if (!selectedItems.length) { setSelectedItems(items); } } else { - setSelectedKeys(items.map(item => item.key).join("|")); + setSelectedKeys(items.map(item => item.key).join(valueSeparator)); setSelectedItems(items); } - }, [selectedKey, selectedKeys, selectedItems, selOptions]); + }, [selectedKey, selectedKeys, selectedItems, selOptions, valueSeparator]); React.useEffect(() => { // only on mount, pre-populate selectedItems from url @@ -170,7 +172,7 @@ const AsyncDropdown: React.FunctionComponent = ({ React.useEffect(() => { if (multiSelect) { if (!selectedItems.length && selectedKey === "") return; - if (selectedItems.map(item => item.key).join("|") === selectedKey) return; + if (selectedItems.map(item => item.key).join(valueSeparator) === selectedKey) return; onChange(undefined, selectedItems, null); } else { if (!selectedItem || selectedItem?.key === selectedKey) return; @@ -178,7 +180,7 @@ const AsyncDropdown: React.FunctionComponent = ({ onChange(undefined, selectedItem, selectedIdx); } } - }, [onChange, multiSelect, selectedItem, selectedIdx, selectedKey, selectedItems]); + }, [onChange, multiSelect, selectedItem, selectedIdx, selectedKey, selectedItems, valueSeparator]); if (multiSelect) { return options === undefined ? @@ -348,11 +350,14 @@ interface QueriesActiveStateField extends BaseField { interface TargetClusterField extends BaseField { type: "target-cluster"; multiSelect?: boolean; + valueSeparator?: string; value?: string; } interface TargetGroupField extends BaseField { type: "target-group"; + multiSelect?: boolean; + valueSeparator?: string; value?: string; } @@ -1133,16 +1138,18 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a break; case "target-cluster": field.value = field.value !== undefined ? field.value : ""; + field.valueSeparator = field.valueSeparator !== undefined ? field.valueSeparator : "|"; retVal.push({ id: fieldID, label: field.label, field: { if (field.multiSelect) { - onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join("|")); + onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join(field.valueSeparator)); } else { onChange(fieldID, (row as IDropdownOption).key); } @@ -1183,6 +1190,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a break; case "target-group": field.value = field.value !== undefined ? field.value : ""; + field.valueSeparator = field.valueSeparator !== undefined ? field.valueSeparator : ","; retVal.push({ id: fieldID, label: field.label, @@ -1190,7 +1198,15 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a key={fieldID} required={field.required} selectedKey={field.value} - onChange={(ev, row: IDropdownOption) => onChange(fieldID, row.key)} + multiSelect={field.multiSelect} + valueSeparator={field.valueSeparator} + onChange={(ev, row) => { + if (field.multiSelect) { + onChange(fieldID, (row as IDropdownOption[]).map(i => i.key).join(field.valueSeparator)); + } else { + onChange(fieldID, (row as IDropdownOption).key); + } + }} placeholder={field.placeholder} /> }); From f5df9ccafa2c378165bde10f9a85a7b2e274b97b Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Wed, 29 May 2024 14:26:34 +0100 Subject: [PATCH 11/18] HPCC-31956 Better error recovery when downgrading systems Signed-off-by: Gavin Halliday --- system/jlib/jlzw.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/system/jlib/jlzw.cpp b/system/jlib/jlzw.cpp index 2dbf48aca62..03f1344f510 100644 --- a/system/jlib/jlzw.cpp +++ b/system/jlib/jlzw.cpp @@ -804,10 +804,13 @@ void decompressToBuffer(MemoryBuffer & out, const void * src) void decompressToBuffer(MemoryBuffer & out, MemoryBuffer & in) { - bool compressed; + unsigned char method; size32_t srcLen; - in.read(compressed).read(srcLen); - if (compressed) + in.read(method).read(srcLen); + if (method > 1) + throw makeStringException(-1, "New compression format is not supported in this version"); + + if (method != 0) decompressToBuffer(out, in.readDirect(srcLen)); else out.append(srcLen, in.readDirect(srcLen)); From a52179f7e1ec9ac3d0c1791823e02e209c30d68c Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Wed, 29 May 2024 16:22:08 +0100 Subject: [PATCH 12/18] HPCC-31958 Internal error from old values file Safer check for components Signed-off-by: Richard Chapman --- helm/hpcc/templates/_warnings.tpl | 40 ++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/helm/hpcc/templates/_warnings.tpl b/helm/hpcc/templates/_warnings.tpl index 3fd828069ec..97756916f09 100644 --- a/helm/hpcc/templates/_warnings.tpl +++ b/helm/hpcc/templates/_warnings.tpl @@ -123,27 +123,29 @@ Pass in dict with root and warnings {{- end -}} {{- range $cname, $ctypes := $ctx.components -}} {{- range $id, $component := $ctypes -}} - {{- if and (kindIs "map" $component) (not $component.disabled) -}} - {{- $hasResources := "" -}} - {{- if eq $cname "thor" -}} - {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.managerResources) -}} - {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.workerResources)) $hasResources -}} - {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.eclAgentResources)) $hasResources -}} - {{- else -}} - {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.resources) -}} - {{- end -}} - {{- if not $hasResources -}} - {{- $_ := set $ctx "missingResources" (append $ctx.missingResources ($component.name | default $id)) -}} - {{- end -}} - {{- /* Checks related to components that are used for cost reporting */ -}} - {{- /* (n.b. cpuRate ignored for components other than thor, eclagent and eclccserver)*/ -}} - {{- if has $cname (list "thor" "eclagent" "eclccserver") -}} - {{- if and $ctx.usingDefaultCpuCost (not $component.cost) -}} - {{- $_ := set $ctx "defaultCpuRateComponents" (append $ctx.defaultCpuRateComponents $component.name) -}} + {{- if (kindIs "map" $component) -}} + {{- if (not $component.disabled) -}} + {{- $hasResources := "" -}} + {{- if eq $cname "thor" -}} + {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.managerResources) -}} + {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.workerResources)) $hasResources -}} + {{- $hasResources = (eq $hasResources "true") | ternary (include "hpcc.hasResources" (dict "resources" $component.eclAgentResources)) $hasResources -}} + {{- else -}} + {{- $hasResources = include "hpcc.hasResources" (dict "resources" $component.resources) -}} {{- end -}} - {{- /* Components that are used for cost reporting require resources: warn if resources missing*/ -}} {{- if not $hasResources -}} - {{- $_ := set $ctx "missingResourcesForCosts" (append $ctx.missingResourcesForCosts ($component.name | default $id)) -}} + {{- $_ := set $ctx "missingResources" (append $ctx.missingResources ($component.name | default $id)) -}} + {{- end -}} + {{- /* Checks related to components that are used for cost reporting */ -}} + {{- /* (n.b. cpuRate ignored for components other than thor, eclagent and eclccserver)*/ -}} + {{- if has $cname (list "thor" "eclagent" "eclccserver") -}} + {{- if and $ctx.usingDefaultCpuCost (not $component.cost) -}} + {{- $_ := set $ctx "defaultCpuRateComponents" (append $ctx.defaultCpuRateComponents $component.name) -}} + {{- end -}} + {{- /* Components that are used for cost reporting require resources: warn if resources missing*/ -}} + {{- if not $hasResources -}} + {{- $_ := set $ctx "missingResourcesForCosts" (append $ctx.missingResourcesForCosts ($component.name | default $id)) -}} + {{- end -}} {{- end -}} {{- end -}} {{- end -}} From 3ad0db1690fdea46b839df554779e945bbd93dc4 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 30 May 2024 12:39:36 +0100 Subject: [PATCH 13/18] HPCC-31961 Remove block size limit for compressed sorted spill files Signed-off-by: Gavin Halliday --- ecl/regress/issue31961.ecl | 44 ++++++++++++++++++++++++++++++++++++++ thorlcr/thorutil/thmem.cpp | 16 +++++++++++--- 2 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 ecl/regress/issue31961.ecl diff --git a/ecl/regress/issue31961.ecl b/ecl/regress/issue31961.ecl new file mode 100644 index 00000000000..7feca48cdd3 --- /dev/null +++ b/ecl/regress/issue31961.ecl @@ -0,0 +1,44 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + + + +strRec := { string x; }; + +myRec := RECORD + STRING a; + STRING b; + STRING c; + STRING d; + DATASET(strRec) e; +END; + + +makeDs(unsigned start) := DATASET(100, TRANSFORM(strRec, SELF.x := (STRING)(start + counter))); + +myRec t(unsigned cnt) := TRANSFORM + SELF.a := (STRING)cnt; + SELF.b := (STRING)HASH32(cnt); + SELF.c := (STRING)HASH64(cnt); + SELF.d := (STRING)HASH64(cnt+1); + SELF.e := makeDs(cnt); +END; + +i := DATASET(1000000, t(COUNTER)); +s := SORT(i, a, HINT(sortCompBlkSz(1000)), LOCAL); +c := COUNT(NOFOLD(s)); +output(c); diff --git a/thorlcr/thorutil/thmem.cpp b/thorlcr/thorutil/thmem.cpp index 77091fcd426..03db3cb8e70 100644 --- a/thorlcr/thorutil/thmem.cpp +++ b/thorlcr/thorutil/thmem.cpp @@ -1862,9 +1862,19 @@ class CThorRowCollectorBase : public CSpillable * if there are a lot of spill files, the merge opens them all and causes excessive * memory usage. */ - size32_t compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, DEFAULT_SORT_COMPBLKSZ); - ActPrintLog(&activity, thorDetailedLogLevel, "%sSpilling will use compressed block size = %u", tracingPrefix.str(), compBlkSz); - spillableRows.setCompBlockSize(compBlkSz); + /* + * However.... HPCC-29385 Changed the way that compressed files are decompressed, so they are only decompressed one + * compression buffer at a time. For LZ4 this means they can only ever expand to the compressed block size (typically 1MB) + * rather than 10s of times that space. + * LZW could still expand many times its block size, but the default for LZW is already 64K which is low enough. + * Therefore only set the compress block size if it has been explicitly set. + */ + size32_t compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, 0); + if (compBlkSz) + { + ActPrintLog(&activity, thorDetailedLogLevel, "%sSpilling will use compressed block size = %u", tracingPrefix.str(), compBlkSz); + spillableRows.setCompBlockSize(compBlkSz); + } } } ~CThorRowCollectorBase() From 46deb4d3f0b612b1fc12999614e8a84a0bef9585 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Fri, 31 May 2024 09:10:24 +0100 Subject: [PATCH 14/18] HPCC-31975 Add k8s service externalTrafficPolicy Add the ability to define externalTrafficPolicy to a service at the service level or within a visibility. Signed-off-by: Jake Smith --- helm/hpcc/templates/_helpers.tpl | 5 +++++ helm/hpcc/values.schema.json | 10 ++++++++++ helm/hpcc/values.yaml | 4 ++++ 3 files changed, 19 insertions(+) diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 51c3f423080..043b0ff4766 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1489,6 +1489,7 @@ Pass in dict with .root, .name, .service, .defaultPort, .selector defined {{- if hasKey $globalServiceInfo "labels" -}}{{- $_ := set $lvars "labels" (merge $lvars.labels $globalServiceInfo.labels) -}}{{- end -}} {{- if hasKey $globalServiceInfo "annotations" -}}{{- $_ := set $lvars "annotations" (merge $lvars.annotations $globalServiceInfo.annotations) -}}{{- end -}} {{- if hasKey $globalServiceInfo "ingress" -}}{{- $_ := set $lvars "ingress" $globalServiceInfo.ingress -}}{{- end -}} + {{- if hasKey $globalServiceInfo "externalTrafficPolicy" -}}{{- $_ := set $lvars "externalTrafficPolicy" $globalServiceInfo.externalTrafficPolicy -}}{{- end -}} {{- if hasKey $globalServiceInfo "loadBalancerSourceRanges" -}}{{- $_ := set $lvars "loadBalancerSourceRanges" $globalServiceInfo.loadBalancerSourceRanges -}}{{- end -}} {{- $_ := set $lvars "type" $globalServiceInfo.type -}} {{- else -}} @@ -1509,6 +1510,7 @@ Pass in dict with .root, .name, .service, .defaultPort, .selector defined {{- end }} {{- end -}} {{- if hasKey .service "ingress" -}}{{- $_ := set $lvars "ingress" .service.ingress -}}{{- end -}} + {{- if hasKey .service "externalTrafficPolicy" -}}{{- $_ := set $lvars "externalTrafficPolicy" .service.externalTrafficPolicy -}}{{- end -}} {{- if hasKey .service "loadBalancerSourceRanges" -}}{{- $_ := set $lvars "loadBalancerSourceRanges" .service.loadBalancerSourceRanges -}}{{- end -}} {{- end }} @@ -1537,6 +1539,9 @@ spec: selector: server: {{ .selector | quote }} type: {{ $lvars.type }} +{{- if $lvars.externalTrafficPolicy }} + externalTrafficPolicy: {{ $lvars.externalTrafficPolicy }} +{{- end }} {{- if $lvars.loadBalancerSourceRanges }} loadBalancerSourceRanges: {{- if ne $lvars.type "LoadBalancer" -}} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index 1f349a65d5a..e028ee27f92 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -404,6 +404,11 @@ "ingress": { "$ref" : "#/definitions/ingress" }, + "externalTrafficPolicy": { + "type": "string", + "enum": ["Cluster", "Local"], + "description": "Route clients to all (Cluster), or local VM only (Local). Preserves client source IPs." + }, "loadBalancerSourceRanges": { "type": "array", "items": { "type": "string" }, @@ -1292,6 +1297,11 @@ "type": "object", "additionalProperties": { "type": "string" } }, + "externalTrafficPolicy": { + "type": "string", + "enum": ["Cluster", "Local"], + "description": "Route clients to all (Cluster), or local VM only (Local). Preserves client source IPs." + }, "loadBalancerSourceRanges": { "type": "array", "items": { "type": "string" }, diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml index d11e0160e51..9889651dbe9 100644 --- a/helm/hpcc/values.yaml +++ b/helm/hpcc/values.yaml @@ -115,6 +115,8 @@ global: type: LoadBalancer ingress: - {} + ## route traffic to local VM only. Also preserves source IPs + #externalTrafficPolicy: Local ## CIDRS allowed to access this service. #loadBalancerSourceRanges: [1.2.3.4/32, 5.6.7.8/32] @@ -636,6 +638,8 @@ esp: # url: "http://abc.com/def?g=1" ## CIDRS allowed to access this service. #loadBalancerSourceRanges: [1.2.3.4/32, 5.6.7.8/32] + ## route traffic to local VM only. Also preserves source IPs + #externalTrafficPolicy: Local # Increase maxRequestEntityLength when query deployments (or similar actions) start to fail because they surpass the maximum size # default for EclWatch is 60M, default for other services is 8M #maxRequestEntityLength: 70M From ba00d0e89658a67d7440a124643a0914122d4467 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Wed, 15 May 2024 11:30:56 +0100 Subject: [PATCH 15/18] HPCC-31940 Add a visual gantt overlay onto the WUs page Signed-off-by: Gordon Smith --- esp/src/src-react/components/Workunits.tsx | 48 +++++++++++++++++-- .../src-react/components/controls/Grid.tsx | 21 +++++--- esp/src/src-react/hooks/platform.ts | 25 +++++++++- esp/src/src/ESPWorkunit.ts | 44 ++++++++++++++++- esp/src/src/Utility.ts | 4 ++ 5 files changed, 129 insertions(+), 13 deletions(-) diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index a00a73e4670..b2047850ee7 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -1,5 +1,6 @@ import * as React from "react"; -import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, Icon, Image, Link } from "@fluentui/react"; +import { CommandBar, ContextualMenuItemType, DetailsRow, ICommandBarItemProps, IDetailsRowProps, Icon, Image, Link } from "@fluentui/react"; +import { hsl as d3Hsl } from "@hpcc-js/common"; import { SizeMe } from "react-sizeme"; import { CreateWUQueryStore, defaultSort, emptyFilter, Get, WUQueryStore, formatQuery } from "src/ESPWorkunit"; import * as WsWorkunits from "src/WsWorkunits"; @@ -7,6 +8,8 @@ import { formatCost } from "src/Session"; import nlsHPCC from "src/nlsHPCC"; import { useConfirm } from "../hooks/confirm"; import { useMyAccount } from "../hooks/user"; +import { useUserStore } from "../hooks/store"; +import { useLogicalClustersPalette } from "../hooks/platform"; import { calcSearch, pushParams } from "../util/history"; import { useHasFocus, useIsMounted } from "../hooks/util"; import { HolyGrail } from "../layouts/HolyGrail"; @@ -61,12 +64,14 @@ export const Workunits: React.FunctionComponent = ({ const [showFilter, setShowFilter] = React.useState(false); const { currentUser } = useMyAccount(); const [uiState, setUIState] = React.useState({ ...defaultUIState }); + const [showTimeline, setShowTimeline] = useUserStore("workunits_showTimeline", true); const { selection, setSelection, pageNum, setPageNum, pageSize, setPageSize, total, setTotal, refreshTable } = useFluentStoreState({ page }); + const [, , palette] = useLogicalClustersPalette(); // Refresh on focus --- const isMounted = useIsMounted(); @@ -243,7 +248,15 @@ export const Workunits: React.FunctionComponent = ({ pushParams(filter); } }, - ], [currentUser, filter, hasFilter, refreshTable, selection, setShowAbortConfirm, setShowDeleteConfirm, store, total, uiState.hasNotCompleted, uiState.hasNotProtected, uiState.hasProtected, uiState.hasSelection]); + { key: "divider_5", itemType: ContextualMenuItemType.Divider, onRender: () => }, + { + key: "timeline", text: nlsHPCC.Timeline, canCheck: true, checked: showTimeline, iconProps: { iconName: "TimelineProgress" }, + onClick: () => { + setShowTimeline(!showTimeline); + refreshTable.call(); + } + }, + ], [currentUser.username, filter, hasFilter, refreshTable, selection, setShowAbortConfirm, setShowDeleteConfirm, setShowTimeline, showTimeline, store, total, uiState.hasNotCompleted, uiState.hasNotProtected, uiState.hasProtected, uiState.hasSelection]); // Selection --- React.useEffect(() => { @@ -274,6 +287,34 @@ export const Workunits: React.FunctionComponent = ({ setUIState(state); }, [selection]); + const renderRowTimings = React.useCallback((props: IDetailsRowProps, size: { readonly width: number; readonly height: number; }) => { + if (showTimeline && props) { + const total = props.item.timings.page.end - props.item.timings.page.start; + const startPct = 100 - (props.item.timings.start - props.item.timings.page.start) / total * 100; + const endPct = 100 - (props.item.timings.end - props.item.timings.page.start) / total * 100; + const backgroundColor = palette(props.item.Cluster); + const borderColor = d3Hsl(backgroundColor).darker().toString(); + + return
+ +
+
; + } + return ; + }, [palette, showTimeline]); + return } main={ @@ -293,6 +334,7 @@ export const Workunits: React.FunctionComponent = ({ setSelection={setSelection} setTotal={setTotal} refresh={refreshTable} + onRenderRow={showTimeline ? props => renderRowTimings(props, size) : undefined} >
@@ -309,7 +351,7 @@ export const Workunits: React.FunctionComponent = ({ setPageNum={setPageNum} setPageSize={setPageSize} total={total} - >} + >} footerStyles={{}} />; }; diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 08ccb145962..fff30e920ca 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -1,5 +1,5 @@ import * as React from "react"; -import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IDetailsList } from "@fluentui/react"; +import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IDetailsList, IRenderFunction, IDetailsRowProps } from "@fluentui/react"; import { Pagination } from "@fluentui/react-experiments/lib/Pagination"; import { useConst, useId, useMount, useOnEvent } from "@fluentui/react-hooks"; import { BaseStore, Memory, QueryRequest, QuerySortItem } from "src/store/Memory"; @@ -193,6 +193,7 @@ interface FluentStoreGridProps { refresh: RefreshTable, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, + onRenderRow?: IRenderFunction } const FluentStoreGrid: React.FunctionComponent = ({ @@ -206,6 +207,7 @@ const FluentStoreGrid: React.FunctionComponent = ({ refresh, setSelection, setTotal, + onRenderRow }) => { const memoizedColumns = useDeepMemo(() => columns, [], [columns]); const [sorted, setSorted] = React.useState(sort); @@ -320,6 +322,7 @@ const FluentStoreGrid: React.FunctionComponent = ({ onColumnHeaderClick={onColumnClick} onRenderDetailsHeader={renderDetailsHeader} onColumnResize={columnResize} + onRenderRow={onRenderRow} styles={gridStyles(height)} /> ; @@ -334,7 +337,8 @@ interface FluentGridProps { height?: string, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, - refresh: RefreshTable + refresh: RefreshTable, + onRenderRow?: IRenderFunction } export const FluentGrid: React.FunctionComponent = ({ @@ -346,7 +350,8 @@ export const FluentGrid: React.FunctionComponent = ({ height, setSelection, setTotal, - refresh + refresh, + onRenderRow }) => { const constStore = useConst(() => new Memory(primaryID, alphaNumColumns)); @@ -357,7 +362,7 @@ export const FluentGrid: React.FunctionComponent = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [constStore, data, /*refresh*/]); - return + return ; }; @@ -372,7 +377,8 @@ interface FluentPagedGridProps { height?: string, setSelection: (selection: any[]) => void, setTotal: (total: number) => void, - refresh: RefreshTable + refresh: RefreshTable, + onRenderRow?: IRenderFunction } export const FluentPagedGrid: React.FunctionComponent = ({ @@ -386,7 +392,8 @@ export const FluentPagedGrid: React.FunctionComponent = ({ height, setSelection, setTotal, - refresh + refresh, + onRenderRow }) => { const [page, setPage] = React.useState(pageNum - 1); const [sortBy, setSortBy] = React.useState(sort); @@ -407,7 +414,7 @@ export const FluentPagedGrid: React.FunctionComponent = ({ setPage(_page); }, [pageNum]); - return + return ; }; diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index 6c634aed36e..8e90b4ab363 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -6,6 +6,7 @@ import { Topology, WsTopology, WorkunitsServiceEx } from "@hpcc-js/comms"; import { getBuildInfo, BuildInfo, fetchModernMode } from "src/Session"; import { cmake_build_type, containerized, ModernMode } from "src/BuildInfo"; import { sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; +import { Palette } from "@hpcc-js/common"; const logger = scopedLogger("src-react/hooks/platform.ts"); @@ -36,14 +37,18 @@ export function useBuildInfo(): [BuildInfo, { isContainer: boolean, currencyCode return [buildInfo, { isContainer, currencyCode, opsCategory }]; } +let g_targetCluster: Promise; export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined, WsTopology.TpLogicalCluster | undefined] { const [targetClusters, setTargetClusters] = React.useState(); const [defaultCluster, setDefaultCluster] = React.useState(); React.useEffect(() => { - const topology = Topology.attach({ baseUrl: "" }); + if (!g_targetCluster) { + const topology = Topology.attach({ baseUrl: "" }); + g_targetCluster = topology.fetchLogicalClusters(); + } let active = true; - topology.fetchLogicalClusters().then(response => { + g_targetCluster.then(response => { if (active) { setTargetClusters(response); let firstRow: WsTopology.TpLogicalCluster; @@ -70,6 +75,22 @@ export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined return [targetClusters, defaultCluster]; } +export function useLogicalClustersPalette(): [WsTopology.TpLogicalCluster[] | undefined, WsTopology.TpLogicalCluster | undefined, Palette.OrdinalPaletteFunc] { + const [targetClusters, defaultCluster] = useLogicalClusters(); + + const palette = useConst(() => Palette.ordinal("workunits", ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd", "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"])); + + React.useEffect(() => { + if (targetClusters) { + targetClusters.forEach(cluster => { + palette(cluster.Name); + }); + } + }, [palette, targetClusters]); + + return [targetClusters, defaultCluster, palette]; +} + let wuCheckFeaturesPromise; export const fetchCheckFeatures = () => { if (!wuCheckFeaturesPromise) { diff --git a/esp/src/src/ESPWorkunit.ts b/esp/src/src/ESPWorkunit.ts index 428eb156107..28b2806b2e3 100644 --- a/esp/src/src/ESPWorkunit.ts +++ b/esp/src/src/ESPWorkunit.ts @@ -1085,8 +1085,50 @@ export function CreateWUQueryStore(): BaseStore { + const page = { + start: undefined, + end: undefined + }; + const data = response.Workunits.ECLWorkunit.map(wu => { + const start = Utility.wuidToDateTime(wu.Wuid); + if (!page.start || page.start > start) { + page.start = start; + } + let timePartsSection = 0; + const end = new Date(start); + const timeParts = wu.TotalClusterTime.split(":"); + while (timeParts.length) { + const timePart = timeParts.pop(); + switch (timePartsSection) { + case 0: + end.setSeconds(end.getSeconds() + +timePart); + break; + case 1: + end.setMinutes(end.getMinutes() + +timePart); + break; + case 2: + end.setHours(end.getHours() + +timePart); + break; + case 3: + end.setDate(end.getDate() + +timePart); + break; + } + ++timePartsSection; + } + if (!page.end || page.end < end) { + page.end = end; + } + return { + ...Get(wu.Wuid, wu), + timings: { + start, + end, + page + } + }; + }); return { - data: response.Workunits.ECLWorkunit.map(wu => Get(wu.Wuid, wu)), + data, total: response.NumWUs }; }); diff --git a/esp/src/src/Utility.ts b/esp/src/src/Utility.ts index e8ee8e05db8..cc78350ed6d 100644 --- a/esp/src/src/Utility.ts +++ b/esp/src/src/Utility.ts @@ -1303,4 +1303,8 @@ export function wuidToDate(wuid: string): string { export function wuidToTime(wuid: string): string { return `${wuid.substring(10, 12)}:${wuid.substring(12, 14)}:${wuid.substring(14, 16)}`; +} + +export function wuidToDateTime(wuid: string): Date { + return new Date(`${wuidToDate(wuid)}T${wuidToTime(wuid)}Z`); } \ No newline at end of file From 96be1d8442701449418216a64aa6dffe8cce7f2f Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Wed, 29 May 2024 14:44:06 +0100 Subject: [PATCH 16/18] HPCC-31957 Remove unnecessary exclusive lock on source file During an XML spray/despray filecopy was unecessarily obtaining an exclusive write lock on the source file attributes. If the file was being read elsewhere, this could cause a deadlock whilst it waited for the exclusive lock. (in one case it was observed that 2 jobs were started performing a despray of the same source file, and they deadlocked each other) Signed-off-by: Jake Smith --- dali/ft/filecopy.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 0668b713f61..ec6f2923559 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -1750,8 +1750,7 @@ void FileSprayer::analyseFileHeaders(bool setcurheadersize) // Despray from distributed file // Check XMLheader/footer in file level - DistributedFilePropertyLock lock(distributedSource); - IPropertyTree &curProps = lock.queryAttributes(); + IPropertyTree &curProps = distributedSource->queryAttributes(); if (curProps.hasProp(FPheaderLength) && curProps.hasProp(FPfooterLength)) { cur.xmlHeaderLength = curProps.getPropInt(FPheaderLength, 0); From d93a42a32b40f1d1c3bed9d75e2ebe65a2b4bde7 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 30 May 2024 19:12:40 +0100 Subject: [PATCH 17/18] HPCC-31961 Use standard compressed block size by default in global sort merger Signed-off-by: Jake Smith --- thorlcr/msort/tsorts.cpp | 15 +++++++++++++-- thorlcr/thorutil/thbufdef.hpp | 1 - 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 41d8b798002..935e1702aa6 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -211,8 +211,19 @@ class CWriteIntercept : public CSimpleInterface rwFlags |= rw_compress; rwFlags |= spillCompInfo; compressedOverflowFile = true; - compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, DEFAULT_SORT_COMPBLKSZ); - ActPrintLog(&activity, "Creating compressed merged overflow file (block size = %u)", compBlkSz); + + /* + * NB: HPCC-29385 Changed the way that compressed files are decompressed, so they are only decompressed one + * compression buffer at a time. For LZ4 this means they can only ever expand to the compressed block size (typically 1MB) + * rather than 10s of times that space. + * LZW could still expand many times its block size, but the default for LZW is already 64K which is low enough. + * Therefore we default to 0 here, which causes createRowWriter to use the default block size for the compressor type. + */ + compBlkSz = activity.getOptUInt(THOROPT_SORT_COMPBLKSZ, 0); + if (compBlkSz) + ActPrintLog(&activity, "Creating compressed merged overflow file (block size = %u)", compBlkSz); + else + ActPrintLog(&activity, "Creating compressed merged overflow file"); } } diff --git a/thorlcr/thorutil/thbufdef.hpp b/thorlcr/thorutil/thbufdef.hpp index bb61498962c..a5291a30364 100644 --- a/thorlcr/thorutil/thbufdef.hpp +++ b/thorlcr/thorutil/thbufdef.hpp @@ -54,7 +54,6 @@ #define EXCESSIVE_PARALLEL_THRESHHOLD (0x500000) // 5MB #define LOOP_SMART_BUFFER_SIZE (0x100000*12) // 12MB #define LOCALRESULT_BUFFER_SIZE (0x100000*10) // 10MB -#define DEFAULT_SORT_COMPBLKSZ (0x10000) // 64K #define DEFAULT_KEYNODECACHEMB 10 #define DEFAULT_KEYLEAFCACHEMB 50 From 811459d7804255c7b70c500ce6847f15e2e061b8 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 30 May 2024 13:03:00 +0100 Subject: [PATCH 18/18] HPCC-31962 Avoid Dali pod overlap with StatefulSet Signed-off-by: Jake Smith --- helm/hpcc/templates/dali.yaml | 13 ++++++++++++- testing/helm/run.sh | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index fc36caa774d..69cea36ac7d 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -67,7 +67,7 @@ true {{- end -}} {{- end }} apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ $dali.name | quote }} spec: @@ -77,6 +77,10 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 0 template: metadata: labels: @@ -97,6 +101,13 @@ spec: {{- include "hpcc.addPrometheusScrapeAnnotations" $.Values.global.metrics | nindent 8 }} {{- end }} spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app: dali + topologyKey: "kubernetes.io/hostname" {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $dali.name "type" "dali") | indent 6 }} serviceAccountName: "hpcc-dali" terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds | default 3600 }} diff --git a/testing/helm/run.sh b/testing/helm/run.sh index d3f2ef17367..27db917d9f6 100755 --- a/testing/helm/run.sh +++ b/testing/helm/run.sh @@ -78,6 +78,7 @@ if type kube-score >/dev/null 2> /dev/null; then --ignore-container-cpu-limit \ --ignore-container-memory-limit \ --ignore-test deployment-has-poddisruptionbudget \ + --ignore-test statefulset-has-poddisruptionbudget \ - >results.txt 2>errors.txt if [ $? -ne 0 ] then