From f811046fd5d841020b59cf06eb20eecb62a66361 Mon Sep 17 00:00:00 2001 From: "Dan S. Camper" Date: Thu, 7 Sep 2023 07:12:29 -0500 Subject: [PATCH 01/14] HPCC-30227 Python Plugin: Fix incorrect custom namedtuple activation Unlikely future problem, if we need to create a new plugin if/when Python moves to a version >= 4.0, but it is possible. Found while looking at other code. Signed-off-by: Dan S. Camper --- plugins/py3embed/py3embed.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/plugins/py3embed/py3embed.cpp b/plugins/py3embed/py3embed.cpp index bf4a45e9fb5..14220d0a352 100644 --- a/plugins/py3embed/py3embed.cpp +++ b/plugins/py3embed/py3embed.cpp @@ -48,11 +48,8 @@ #define Py_TPFLAGS_HAVE_ITER 0 #endif -#if PY_MINOR_VERSION < 7 - #define USE_CUSTOM_NAMEDTUPLES -#endif - #if PY_VERSION_HEX < 0x03070000 + #define USE_CUSTOM_NAMEDTUPLES #define INIT_PY_THREADS #endif From 45b9b9d0f4411b8eb46bbf0a0a3ff49910418085 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 14 Sep 2023 14:46:59 +0100 Subject: [PATCH 02/14] HPCC-30292 Remove spurious podName logging Signed-off-by: Jake Smith --- system/jlib/jcontainerized.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/system/jlib/jcontainerized.cpp b/system/jlib/jcontainerized.cpp index 0db335c1a4c..c4172e4828a 100644 --- a/system/jlib/jcontainerized.cpp +++ b/system/jlib/jcontainerized.cpp @@ -254,7 +254,6 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) return; // process pod information from environment getEnvVar("MY_POD_NAME", myPodName.clear()); - PROGLOG("The podName = %s", myPodName.str()); }; if (isContainerized()) podInfoInitCBId = installConfigUpdateHook(updateFunc, true); From e4c3f62935ff5fd2dad38656def2a6e2409d4e60 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:34:38 -0400 Subject: [PATCH 03/14] HPCC-30222 ECL Watch cut back multiple calls to WsLogaccess.GetLogs Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/WorkunitDetails.tsx | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/esp/src/src-react/components/WorkunitDetails.tsx b/esp/src/src-react/components/WorkunitDetails.tsx index f8f66be72b8..f1fe4d7ee13 100644 --- a/esp/src/src-react/components/WorkunitDetails.tsx +++ b/esp/src/src-react/components/WorkunitDetails.tsx @@ -1,10 +1,12 @@ import * as React from "react"; import { IPivotItemProps, Pivot, PivotItem } from "@fluentui/react"; +import { scopedLogger } from "@hpcc-js/util"; import { SizeMe } from "react-sizeme"; import nlsHPCC from "src/nlsHPCC"; import { service, hasLogAccess } from "src/ESPLog"; import { useWorkunit } from "../hooks/workunit"; import { useUserTheme } from "../hooks/theme"; +import { useDeepEffect } from "../hooks/deepHooks"; import { DojoAdapter } from "../layouts/DojoAdapter"; import { pivotItemStyle } from "../layouts/pivot"; import { pushUrl } from "../util/history"; @@ -22,6 +24,8 @@ import { WorkunitSummary } from "./WorkunitSummary"; import { Result } from "./Result"; import { Logs } from "./Logs"; +const logger = scopedLogger("src-react/components/WorkunitDetails.tsx"); + interface WorkunitDetailsProps { wuid: string; tab?: string; @@ -54,7 +58,8 @@ export const WorkunitDetails: React.FunctionComponent = ({ const [logCount, setLogCount] = React.useState("*"); const [logsDisabled, setLogsDisabled] = React.useState(true); - React.useEffect(() => { + + useDeepEffect(() => { hasLogAccess().then(response => { setLogsDisabled(!response); return response; @@ -62,12 +67,12 @@ export const WorkunitDetails: React.FunctionComponent = ({ if (hasLogAccess) { service.GetLogsEx({ ...queryParams, jobId: wuid, LogLineStartFrom: 0, LogLineLimit: 10 }).then(response => { // HPCC-27711 - Requesting LogLineLimit=1 causes issues setLogCount(response.total); - }); + }).catch((err) => logger.error(err)); } }).catch(() => { setLogsDisabled(true); }); - }, [queryParams, wuid]); + }, [wuid], [queryParams]); return {({ size }) => pushUrl(`/workunits/${wuid}/${evt.props.itemKey}`)}> From 48125181667844c0d978be7d2dbea160210fe396 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 14 Sep 2023 17:25:13 +0100 Subject: [PATCH 04/14] Split off 9.0.44 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index ddf646b855d..1ac81bfe533 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.0.43-closedown0 +version: 9.0.45-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.0.43-closedown0 +appVersion: 9.0.45-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 70ee98883e6..11789159890 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1314,7 +1314,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index d26ae5b3625..0ce4da74ba7 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 711ab785db5..979b27a8f67 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index d83eaeb6788..7b256901ce3 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 19830ee09a0..04f5b8bfb3e 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index cbba2ba22f2..2022c3c19fe 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index ebcb7fa27c9..9adfb6883b2 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index cfffa85852c..d56547933cd 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 8b7cbf24ada..a1ce07aee1d 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index e23c02bc1c3..e8420fd3008 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index e28d5adc9be..1fbb14d81ba 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index cb4a7ffa2d9..ebaa8b39a8f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.0.43-closedown0 + helmVersion: 9.0.45-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index e25b863f699..d6726e752c2 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 0 ) -set ( HPCC_POINT 43 ) +set ( HPCC_POINT 45 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 250e730ad9b88303b81e14391d1c217ebc866047 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Thu, 14 Sep 2023 17:28:26 +0100 Subject: [PATCH 05/14] Split off 9.2.22 Signed-off-by: Jake Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 8ac68a684aa..3ebdf68242d 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.21-closedown0 +version: 9.2.23-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.21-closedown0 +appVersion: 9.2.23-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index c9da7756f8d..678d23a5d9b 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1337,7 +1337,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 1e1b36c788d..63c57a67272 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 998d0d43373..30a0997eb70 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index 7f61b7b3a96..e66afba9943 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index be74659f375..02e425c200e 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 30aa1dae501..410432b94f6 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index 99a6fcd9503..d9e2a4209da 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index ca7647d2734..c7882f93e4f 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 7c703e1a7fe..0b46ae47763 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index c0f10d3209a..9f6877525cf 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index 2f669a3cdb7..c7edfeaf960 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 4e578687dc7..27044dad14f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.21-closedown0 + helmVersion: 9.2.23-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 3f50c2ba181..5da52977821 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 21 ) +set ( HPCC_POINT 23 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From 3f39ca5f8d134763ac698ebcfff700d52cafff0e Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Thu, 14 Sep 2023 14:10:54 -0400 Subject: [PATCH 06/14] HPCC-30200 HpccInternal scope should be accessible by HPCC Admins Changed check for hpccinternal scope to also allow full access to superusers Signed-Off-By: Kenneth.Rowland@lexisnexisrisk.com --- system/security/LdapSecurity/ldapsecurity.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/security/LdapSecurity/ldapsecurity.cpp b/system/security/LdapSecurity/ldapsecurity.cpp index 2c13708e7b9..7bd62e4e3d0 100644 --- a/system/security/LdapSecurity/ldapsecurity.cpp +++ b/system/security/LdapSecurity/ldapsecurity.cpp @@ -1023,7 +1023,7 @@ SecAccessFlags CLdapSecManager::authorizeFileScope(ISecUser & user, const char * StringBuffer userName; for (const char * p = &filescope[m_hpccInternalScope.length()]; *p && *p != ':'; p++)//extract scope username userName.append(*p); - if(strieq(userName.str(), user.getName())) + if(strieq(userName.str(), user.getName()) || isSuperUser(&user)) return SecAccess_Full; PROGLOG("Access denied to scope %s for user %s", filescope, user.getName()); return SecAccess_None; From a164dc0ffaf8428da31472d6217974968005938e Mon Sep 17 00:00:00 2001 From: Jim DeFabia Date: Wed, 13 Sep 2023 13:52:21 -0400 Subject: [PATCH 07/14] HPCC-30093 Fix doc example to use secrets Signed-off-by: Jim DeFabia --- .../ECLR_mods/BltInFunc-SOAPCALL.xml | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml index f048d319d72..8eaf63b23af 100644 --- a/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml +++ b/docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml @@ -69,7 +69,7 @@ form usernames and passwords, if required. If calling an ESP Web service, you can append the ver_=n.nn parameter to specify the version of the service. For example: SOAPCALL('http://127.0.0.1:8010/Wsdfu/?ver_=1.22', + language="ECL" role="notrunnable">SOAPCALL('https://eclwatch.example.com:8010/Wsdfu/?ver_=1.22', 'DFUSearchData', instructure,DATASET(outsructure)); @@ -444,10 +444,18 @@ STRING500 OutData{XPATH('OutData')}; UNSIGNED4 Latency{XPATH('_call_latency')}; END; -ip := 'http://127.0.0.1:8022/'; -ips := 'https://127.0.0.1:8022/'; -ipspw := 'https://username:password@127.0.0.1:8022/'; +ip := 'http://service.example.com:8022/'; +ips := 'https://service.example.com:8022/'; svc := 'MyModule.SomeService'; +ips_secret := 'secret:myConnectSecret'; + /* assumes a secret named http-connect-myConnectSecret exists & contains: + /* + { + "url": "https://service.example.com:8022/", + "username": "username", + "password": "password" + } + */ //1 rec in, 1 rec out OneRec1 := SOAPCALL(ips,svc,{STRING500 InData := 'Some Input Data'},OutRec1); @@ -460,7 +468,7 @@ OneRec2 := SOAPCALL(InputDataset,ip,svc,{STRING500 InData},OutRec1); //recordset in, recordset out ManyRec2 := - SOAPCALL(InputDataset,ipspw,svc,{STRING500 InData := 'Some In Data'},DATASET(OutRec1)); + SOAPCALL(InputDataset,ips_secret,svc,{STRING500 InData := 'Some In Data'},DATASET(OutRec1)); //TRANSFORM function usage example namesRecord := RECORD @@ -494,20 +502,17 @@ outRecord genDefault2(namesRecord l) := TRANSFORM SELF.score := 0; END; -ip := 'http://127.0.0.1:8022/'; -svc:= 'MyModule.SomeService'; OUTPUT(SOAPCALL(ip, svc,{ STRING20 surname := 'Halligan',STRING20 forename := 'Kevin';}, + DATASET(outRecord), ONFAIL(genDefault1()))); OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(genDefault2(LEFT)))); - OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP))); //Using HTTPHEADER to pass Authorization info OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP), HTTPHEADER('Authorization','Basic dXNlcm5hbWU6cGFzc3dvcmQ='), HTTPHEADER('MyLiteral','FOO'))); - @@ -524,11 +529,11 @@ OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP), Example: //1 rec in, no result -SOAPCALL( 'https://127.0.0.1:8022/','MyModule.SomeService', +SOAPCALL( 'https://service.example.com:8022/','MyModule.SomeService', {STRING500 InData := 'Some Input Data'}); //recordset in, no result -SOAPCALL( InputDataset,'https://127.0.0.1:8022/','MyModule.SomeService',{STRING500 InData}); +SOAPCALL( InputDataset,'https://service.example.com:8022/','MyModule.SomeService',{STRING500 InData}); See Also: RECORD Structure, From 55c65bbb03ff2f47cf53051c06a5104d3b3344fc Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 15 Sep 2023 10:06:14 +0100 Subject: [PATCH 08/14] HPCC-30297 WU Details Results Regression Signed-off-by: Gordon Smith --- esp/src/src-react/components/controls/Grid.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index c48dc9150eb..41c6c499ee6 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -43,16 +43,16 @@ interface IColumn extends _IColumn { function tooltipItemRenderer(item: any, index: number, column: IColumn) { const id = `${column.key}-${index}`; - const value = item[column.fieldName || column.key] ?? ""; + const value = item[column.fieldName || column.key]; const className = column.data.className ? column.data.className(value, item) : ""; const style: React.CSSProperties = { display: "flex", justifyContent: column.data.justify === "right" ? "flex-end" : "flex-start" }; - return + return {column.data.formatter ? - {column.data.formatter(value, item)} : - {value} + {column.data.formatter(value, item) ?? ""} : + {value ?? ""} } ; } From a66eef636861ab67076df7019c5e61e9c569788b Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 15 Sep 2023 11:19:34 -0400 Subject: [PATCH 09/14] HPCC-29244 ECL Watch v9 fix cannot set job name in playground original change for 29244 didn't fix the issue, because the jobname needed to be submitted to WUUpdate Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/ECLPlayground.tsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/esp/src/src-react/components/ECLPlayground.tsx b/esp/src/src-react/components/ECLPlayground.tsx index f517bdfd37e..7c3d436c02c 100644 --- a/esp/src/src-react/components/ECLPlayground.tsx +++ b/esp/src/src-react/components/ECLPlayground.tsx @@ -74,6 +74,12 @@ const playgroundStyles = mergeStyleSets({ borderLeft: borderStyle, borderRight: borderStyle } + }, + ".ms-Label": { + marginRight: "12px" + }, + ".ms-Label::after": { + paddingRight: 0 } }, }, @@ -122,9 +128,6 @@ const playgroundStyles = mergeStyleSets({ display: "flex", marginLeft: "18px" }, - ".is-disabled .ms-Label": { - paddingRight: "12px" - }, ".ms-TextField-errorMessage": { display: "none" } @@ -231,11 +234,11 @@ const ECLEditorToolbar: React.FunctionComponent = ({ const submitWU = React.useCallback(async () => { const wu = await Workunit.create({ baseUrl: "" }); - await wu.update({ QueryText: editor.ecl() }); + await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster); wu.watchUntilComplete(changes => playgroundResults(wu)); - }, [cluster, editor, playgroundResults]); + }, [cluster, editor, playgroundResults, queryName]); const publishWU = React.useCallback(async () => { if (queryName === "") { @@ -246,7 +249,7 @@ const ECLEditorToolbar: React.FunctionComponent = ({ const wu = await Workunit.create({ baseUrl: "" }); - await wu.update({ QueryText: editor.ecl() }); + await wu.update({ Jobname: queryName, QueryText: editor.ecl() }); await wu.submit(cluster, WUUpdate.Action.Compile); wu.watchUntilComplete(changes => playgroundResults(wu, "publish")); @@ -302,7 +305,6 @@ const ECLEditorToolbar: React.FunctionComponent = ({ Date: Thu, 14 Sep 2023 16:13:02 +0100 Subject: [PATCH 10/14] HPCC-30288 Handle abort race condition that caused crash When a graph aborts it propagates the abort to the workers, and then collects final progress data (new since HPCC-29284). The progress collection might not complete before the worker graph unwound and was removed. The net result was that no progress was sent from the worker to the master in some cases, instead an incompatible empty serialization format was sent back, which caused the manager to crash. This fix waits for the manager to signal 'done' even when aborting for a period, and fixes the deserialization handling if the worker sends no info. Signed-off-by: Jake Smith --- thorlcr/graph/thgraphmaster.cpp | 2 ++ thorlcr/graph/thgraphslave.cpp | 13 ++++++++++--- thorlcr/slave/slavmain.cpp | 4 +++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index 7f4a8e940e0..fe7ed508848 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -2802,6 +2802,8 @@ void CMasterGraph::getFinalProgress(bool aborting) { unsigned slave; msg.read(slave); + if (RANK_NULL == slave) // worker no longer had graph registered + continue; handleSlaveDone(slave, msg); if (!queryOwner()) { diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 5f29b2c5c3b..cf6aa750fa5 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1232,8 +1232,16 @@ void CSlaveGraph::done() GraphPrintLog("End of sub-graph"); progressActive.store(false); setProgressUpdated(); // NB: ensure collected after end of graph - if (!aborted && graphDone && (!queryOwner() || isGlobal())) - getDoneSem.wait(); // must wait on master + if (!queryOwner() || isGlobal()) + { + if (aborted || !graphDone) + { + if (!getDoneSem.wait(SHORTTIMEOUT)) // wait on master to clear up, gather info from slaves + WARNLOG("CSlaveGraph::done - timedout waiting for master to signal done()"); + } + else + getDoneSem.wait(); + } if (!queryOwner()) { if (globals->getPropBool("@watchdogProgressEnabled")) @@ -1485,7 +1493,6 @@ IThorResult *CSlaveGraph::getGlobalResult(CActivityBase &activity, IThorRowInter return result.getClear(); } - /////////////////////////// class CThorCodeContextSlave : public CThorCodeContextBase, implements IEngineContext diff --git a/thorlcr/slave/slavmain.cpp b/thorlcr/slave/slavmain.cpp index 9ffec6a4024..7ba4bf21838 100644 --- a/thorlcr/slave/slavmain.cpp +++ b/thorlcr/slave/slavmain.cpp @@ -2040,7 +2040,9 @@ class CJobListener : public CSimpleInterface } else { - msg.append((rank_t)0); // JCSMORE - not sure why this would ever happen + // implies graph started on master, but aborted, wound-up and was removed from channel + // before GraphEnd/getFinalProgress was issued to this worker. + msg.append(RANK_NULL); // signal to manager there is no info. } } job->reportGraphEnd(gid); From 3717a31056f52c91c89f293d76b50cc9a9774236 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 15 Sep 2023 12:43:37 -0400 Subject: [PATCH 11/14] HPCC-30264 ECL Watch v9 fix Filter failure on Landing Zone original fix for 30264 (merged into 9.0.x) clashed with a feature added by HPCC-29400 (merged into 9.2.x) Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/eclwatch/LZBrowseWidget.js | 9 +++++++++ esp/src/src-react/components/LandingZone.tsx | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/esp/src/eclwatch/LZBrowseWidget.js b/esp/src/eclwatch/LZBrowseWidget.js index c267aa53497..34e62d5b4b6 100644 --- a/esp/src/eclwatch/LZBrowseWidget.js +++ b/esp/src/eclwatch/LZBrowseWidget.js @@ -72,6 +72,8 @@ define([ serverFilterSelect: null, replicateEnabled: null, + dzExpanded: "", + postCreate: function (args) { this.inherited(arguments); this.sprayFixedForm = registry.byId(this.id + "SprayFixedForm"); @@ -708,6 +710,13 @@ define([ displayName: tree({ label: this.i18n.Name, sortable: false, + shouldExpand: function (row, level) { + if ((context.dzExpanded === "" || context.dzExpanded === row.data.DropZone?.Name) && level <= 1) { + context.dzExpanded = row.data.DropZone.Name; + return true; + } + return false; + }, formatter: function (_name, row) { var img = ""; var name = _name; diff --git a/esp/src/src-react/components/LandingZone.tsx b/esp/src/src-react/components/LandingZone.tsx index 370a0cfe4d1..ea19f848aad 100644 --- a/esp/src/src-react/components/LandingZone.tsx +++ b/esp/src/src-react/components/LandingZone.tsx @@ -74,6 +74,8 @@ interface LandingZoneProps { filter?: LandingZoneFilter; } +let dzExpanded = ""; + export const LandingZone: React.FunctionComponent = ({ filter = emptyFilter }) => { @@ -113,7 +115,7 @@ export const LandingZone: React.FunctionComponent = ({ filename: "landingZones", getSelected: function () { if (filter?.__dropZone) { - return this.inherited(arguments, [FileSpray.CreateLandingZonesFilterStore({})]); + return this.inherited(arguments, [FileSpray.CreateLandingZonesFilterStore({ dropZone: filter.__dropZone })]); } return this.inherited(arguments, [FileSpray.CreateFileListStore({})]); }, @@ -136,6 +138,13 @@ export const LandingZone: React.FunctionComponent = ({ displayName: tree({ label: nlsHPCC.Name, sortable: false, + shouldExpand: function (row, level) { + if ((dzExpanded === "" || dzExpanded === row.data.DropZone?.Name) && level <= 1) { + dzExpanded = row.data.DropZone.Name; + return true; + } + return false; + }, formatter: function (_name, row) { let img = ""; let name = row.displayName; From 67904d64b4d0327dff3bf5b2dca35f2fa782c0ff Mon Sep 17 00:00:00 2001 From: g-pan Date: Tue, 5 Sep 2023 18:57:01 -0400 Subject: [PATCH 12/14] HPCC-22273 Document CentOS dependency installation Signed-off-by: g-pan --- .../Installing_and_RunningTheHPCCPlatform.xml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml index fc992890d64..f3f09675266 100644 --- a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml +++ b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml @@ -278,15 +278,18 @@ CentOS/Red Hat - To install the Platform you should have the appropriate - permissions to install packages. If you have sudo rights, then you - can install the platform using yum. + To install the HPCC Systems Platform you should have the + appropriate rights and permissions to install packages on your + system. + + One way to install the platform is using yum. - sudo yum install <hpccsystems platform rpm package> + sudo yum install -y epel-release +sudo yum install -y <hpccsystems platform rpm package> - Optionally you can install the package with rpm (recommended - using the -Uvh options), however then you would have to negotiate - installing any additional dependencies. + Optionally you can install packages with rpm (recommended + using the -Uvh options), however you would then have to negotiate + installing any dependencies. From cb1ed4c909feadacd0c443824caf8bbc980a01d9 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 15 Sep 2023 16:41:19 +0100 Subject: [PATCH 13/14] HPCC-30303 Persist columns widths per page Fix ability to store "any" state on a browser history item Signed-off-by: Gordon Smith --- esp/src/src-react/components/Files.tsx | 1 + esp/src/src-react/components/Title.tsx | 2 +- .../src-react/components/controls/Grid.tsx | 26 ++++++--- esp/src/src-react/components/forms/Login.tsx | 2 +- esp/src/src-react/hooks/store.ts | 53 ++++++++++++++++-- esp/src/src-react/util/history.ts | 55 +++++++++++-------- 6 files changed, 104 insertions(+), 35 deletions(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 907327039a6..03f60dff097 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -160,6 +160,7 @@ export const Files: React.FunctionComponent = ({ }, Name: { label: nlsHPCC.LogicalName, + width: 360, formatter: (name, row) => { const file = Get(row.NodeGroup, name, row); if (row.__hpcc_isDir) { diff --git a/esp/src/src-react/components/Title.tsx b/esp/src/src-react/components/Title.tsx index 510b8fb17ce..b482d08fadd 100644 --- a/esp/src/src-react/components/Title.tsx +++ b/esp/src/src-react/components/Title.tsx @@ -135,7 +135,7 @@ export const DevTitle: React.FunctionComponent = ({ method: "post" }).then(() => { setUserSession({ ...userSession, Status: "Locked" }); - replaceUrl("/login", null, true); + replaceUrl("/login", true); }); } }, diff --git a/esp/src/src-react/components/controls/Grid.tsx b/esp/src/src-react/components/controls/Grid.tsx index 41c6c499ee6..52cc86c8d27 100644 --- a/esp/src/src-react/components/controls/Grid.tsx +++ b/esp/src/src-react/components/controls/Grid.tsx @@ -7,7 +7,7 @@ import nlsHPCC from "src/nlsHPCC"; import { createCopyDownloadSelection } from "../Common"; import { updatePage, updateSort } from "../../util/history"; import { useDeepCallback, useDeepEffect, useDeepMemo } from "../../hooks/deepHooks"; -import { useUserStore } from "../../hooks/store"; +import { useUserStore, useNonReactiveEphemeralPageStore } from "../../hooks/store"; import { useUserTheme } from "../../hooks/theme"; /* --- Debugging dependency changes --- @@ -65,24 +65,30 @@ function updateColumnSorted(columns: IColumn[], attr: any, desc: boolean) { } } -function columnsAdapter(columns: FluentColumns): IColumn[] { +function columnsAdapter(columns: FluentColumns, columnWidths: Map): IColumn[] { const retVal: IColumn[] = []; for (const key in columns) { const column = columns[key]; + const width = columnWidths.get(key) ?? column.width; if (column?.selectorType === undefined && column?.hidden !== true) { retVal.push({ key, name: column.label ?? key, fieldName: column.field ?? key, - minWidth: column.width ?? 70, - maxWidth: column.width, + minWidth: width ?? 70, + maxWidth: width, isResizable: true, isSorted: false, isSortedDescending: false, iconName: column.headerIcon, isIconOnly: !!column.headerIcon, data: column, - onRender: tooltipItemRenderer + styles: { root: { width } }, + onRender: (item: any, index: number, col: IColumn) => { + col.minWidth = column.width ?? 70; + col.maxWidth = column.width; + return tooltipItemRenderer(item, index, col); + } } as IColumn); } } @@ -187,6 +193,7 @@ const FluentStoreGrid: React.FunctionComponent = ({ const memoizedColumns = useDeepMemo(() => columns, [], [columns]); const [sorted, setSorted] = React.useState(sort); const [items, setItems] = React.useState([]); + const [columnWidths] = useNonReactiveEphemeralPageStore("columnWidths"); const selectionHandler = useConst(new Selection({ onSelectionChanged: () => { @@ -220,8 +227,8 @@ const FluentStoreGrid: React.FunctionComponent = ({ }, [], [sort]); const fluentColumns: IColumn[] = React.useMemo(() => { - return columnsAdapter(memoizedColumns); - }, [memoizedColumns]); + return columnsAdapter(memoizedColumns, columnWidths); + }, [columnWidths, memoizedColumns]); React.useEffect(() => { updateColumnSorted(fluentColumns, sorted?.attribute as string, sorted?.descending); @@ -258,6 +265,10 @@ const FluentStoreGrid: React.FunctionComponent = ({ }); }, []); + const columnResize = React.useCallback((column: IColumn, newWidth: number, columnIndex?: number) => { + columnWidths.set(column.key, newWidth); + }, [columnWidths]); + return = ({ selectionPreservedOnEmptyClick={true} onColumnHeaderClick={onColumnClick} onRenderDetailsHeader={renderDetailsHeader} + onColumnResize={columnResize} styles={gridStyles(height)} />; }; diff --git a/esp/src/src-react/components/forms/Login.tsx b/esp/src/src-react/components/forms/Login.tsx index a5dc021cb51..ecf06925c62 100644 --- a/esp/src/src-react/components/forms/Login.tsx +++ b/esp/src/src-react/components/forms/Login.tsx @@ -109,7 +109,7 @@ export const Login: React.FunctionComponent = ({ } else { createUserSession(cookies).then(() => { setErrorMessage(""); - replaceUrl("/", null, true); + replaceUrl("/", true); }).catch(err => logger.error("Unable to create user session.")); } } diff --git a/esp/src/src-react/hooks/store.ts b/esp/src/src-react/hooks/store.ts index 663c6e963d1..e8f598ca566 100644 --- a/esp/src/src-react/hooks/store.ts +++ b/esp/src/src-react/hooks/store.ts @@ -1,6 +1,7 @@ import * as React from "react"; import { useConst } from "@fluentui/react-hooks"; -import { globalKeyValStore, IKeyValStore, userKeyValStore } from "src/KeyValStore"; +import { globalKeyValStore, IKeyValStore, localKeyValStore, sessionKeyValStore, userKeyValStore } from "src/KeyValStore"; +import { parseHash } from "../util/history"; function toString(value: T, defaultValue: T): string { if (value === undefined) value = defaultValue; @@ -79,14 +80,58 @@ function useStore(store: IKeyValStore, key: string, defaultValue: T, monitor: return [value, extSetValue, reset]; } -export function useUserStore(key: string, defaultValue: T, monitor: boolean = false) { +export function useGlobalStore(key: string, defaultValue: T, monitor: boolean = false) { + const store = useConst(() => globalKeyValStore()); + return useStore(store, key, defaultValue, monitor); +} +export function useUserStore(key: string, defaultValue: T, monitor: boolean = false) { const store = useConst(() => userKeyValStore()); return useStore(store, key, defaultValue, monitor); } -export function useGlobalStore(key: string, defaultValue: T, monitor: boolean = false) { +export function useLocalStore(key: string, defaultValue: T, monitor: boolean = false) { + const store = useConst(() => localKeyValStore()); + return useStore(store, key, defaultValue, monitor); +} - const store = useConst(() => globalKeyValStore()); +export function useSessionStore(key: string, defaultValue: T, monitor: boolean = false) { + const store = useConst(() => sessionKeyValStore()); return useStore(store, key, defaultValue, monitor); } + +/* Ephemeral Store + This store is used to persist data that is only needed for the current page. + It is only persisted in a global variable. + It is also non reactive = i.e. changing its content will not trigger a re-render. +*/ + +const g_state: Map>> = new Map(); + +function useNonReactiveEphemeralPageGlobalStore(): [Map>, () => void] { + const pathname = useConst(() => parseHash(window.location.hash).pathname); + + const reset = React.useCallback(() => { + g_state.set(pathname, new Map()); + }, [pathname]); + + if (!g_state.has(pathname)) { + reset(); + } + + return [g_state.get(pathname), reset]; +} + +export function useNonReactiveEphemeralPageStore(id: string): [Map, () => void] { + const [pageStates] = useNonReactiveEphemeralPageGlobalStore(); + + const reset = React.useCallback(() => { + pageStates.set(id, new Map()); + }, [id, pageStates]); + + if (!pageStates.has(id)) { + reset(); + } + + return [pageStates.get(id), reset]; +} diff --git a/esp/src/src-react/util/history.ts b/esp/src/src-react/util/history.ts index 280ff2cb8c8..9521b9902f4 100644 --- a/esp/src/src-react/util/history.ts +++ b/esp/src/src-react/util/history.ts @@ -20,7 +20,7 @@ export function resolve(pathnameOrContext: string | ResolveContext) { return g_router.resolve(pathnameOrContext); } -function parseHash(hash: string): HistoryLocation { +export function parseHash(hash: string): HistoryLocation { if (hash[0] !== "#") { return { pathname: "/", @@ -82,6 +82,7 @@ interface HistoryLocation { pathname: string; search: string; id: string; + state?: { [key: string]: any } } export type ListenerCallback = (location: HistoryLocation, action: string) => void; @@ -140,19 +141,19 @@ class History { return hashUrl; } - push(to: { pathname?: string, search?: string }, state?: S) { + push(to: { pathname?: string, search?: string }) { const newHash = this.fixHash(`${this.trimRightSlash(to.pathname || this.location.pathname)}${to.search || ""}`); if (window.location.hash !== newHash) { - globalHistory.pushState(state, "", newHash); + globalHistory.pushState(undefined, "", newHash); this.location = parseHash(newHash); this.broadcast("PUSH"); } } - replace(to: { pathname?: string, search?: string }, state?: S) { + replace(to: { pathname?: string, search?: string }) { const newHash = this.fixHash(`${this.trimRightSlash(to.pathname || this.location.pathname)}${to.search || ""}`); if (window.location.hash !== newHash) { - globalHistory.replaceState(state, "", newHash); + globalHistory.replaceState(globalHistory.state, "", newHash); this.location = parseHash(newHash); this.broadcast("REPLACE"); } @@ -188,49 +189,49 @@ class History { this.updateRecent(); for (const key in this._listeners) { const listener = this._listeners[key]; - listener(this.location, action); + listener({ ...this.location, state: { ...globalHistory.state } }, action); } } } export const hashHistory = new History(); -export function pushSearch(_: object, state?: any) { +export function pushSearch(_: object) { const search = stringify(_ as any); hashHistory.push({ search: search ? "?" + search : "" - }, state); + }); } -export function updateSearch(_: object, state?: any) { +export function updateSearch(_: object) { const search = stringify(_ as any); hashHistory.replace({ search: search ? "?" + search : "" - }, state); + }); } -export function pushUrl(_: string, state?: any) { +export function pushUrl(_: string) { hashHistory.push({ pathname: _ - }, state); + }); } -export function replaceUrl(_: string, state?: any, refresh: boolean = false) { +export function replaceUrl(_: string, refresh: boolean = false) { hashHistory.replace({ pathname: _ - }, state); + }); if (refresh) window.location.reload(); } -export function pushParam(key: string, val?: string | string[] | number | boolean, state?: any) { - pushParams({ [key]: val }, state); +export function pushParam(key: string, val?: string | string[] | number | boolean) { + pushParams({ [key]: val }); } -export function pushParamExact(key: string, val?: string | string[] | number | boolean, state?: any) { - pushParams({ [key]: val }, state, true); +export function pushParamExact(key: string, val?: string | string[] | number | boolean) { + pushParams({ [key]: val }, true); } -export function pushParams(search: { [key: string]: string | string[] | number | boolean }, state?: any, keepEmpty: boolean = false) { +export function pushParams(search: { [key: string]: string | string[] | number | boolean }, keepEmpty: boolean = false) { const params = parseQuery(hashHistory.location.search); for (const key in search) { const val = search[key]; @@ -241,15 +242,25 @@ export function pushParams(search: { [key: string]: string | string[] | number | params[key] = val; } } - pushSearch(params, state); + pushSearch(params); } -export function updateParam(key: string, val?: string | string[] | number | boolean, state?: any) { +export function updateParam(key: string, val?: string | string[] | number | boolean) { const params = parseQuery(hashHistory.location.search); if (val === undefined) { delete params[key]; } else { params[key] = val; } - updateSearch(params, state); + updateSearch(params); +} + +export function updateState(key: string, val?: string | string[] | number | boolean) { + const state = { ...globalHistory.state }; + if (val === undefined) { + delete state[key]; + } else { + state[key] = val; + } + globalHistory.replaceState(state, ""); } From 23f2e61ac1d69303c216c101e97c1a14b812f6a6 Mon Sep 17 00:00:00 2001 From: wangkx Date: Thu, 7 Sep 2023 08:06:16 -0400 Subject: [PATCH 14/14] HPCC-28499 Add roxie stats to WsECL GUI 1. Display checkbox 'Save stats to workunit.' and checkbox 'Get summary stats.' in WsECL GUI. 2. Pass the requests of the 2 checkboxes to WsECL and roxie. 3. Add code to report the summary stats in roxie when the request is sent from WsECL GUI. 4. Display the stats workunit id and the summary stats in WsECL GUI. Revise based on review: 1. Rename the 'save_stats' to '@statsToWorkunit' and rename the 'summary_stats' to 'summaryStats'. 2. Remove the code for converting the 'save_stats' to the '@statsToWorkunit'. The code is not needed. 3. Add the code to pass the '@statsToWorkunit' and '@summaryStats' to the SOAP test form and JSON test form. Signed-off-by: wangkx --- common/wuwebview/wuwebview.cpp | 2 ++ esp/services/common/jsonhelpers.hpp | 6 ++++++ esp/services/ws_ecl/ws_ecl_service.cpp | 6 ++++++ esp/xslt/wsecl3_form.xsl | 8 ++++++++ esp/xslt/wsecl3_result.xslt | 18 ++++++++++++++++++ roxie/ccd/ccdlistener.cpp | 22 ++++++++++++++++++---- 6 files changed, 58 insertions(+), 4 deletions(-) diff --git a/common/wuwebview/wuwebview.cpp b/common/wuwebview/wuwebview.cpp index fb824ccc042..f74f7bdd242 100644 --- a/common/wuwebview/wuwebview.cpp +++ b/common/wuwebview/wuwebview.cpp @@ -80,6 +80,8 @@ class WuExpandedResultBuffer : implements IPTreeNotifyEvent, public CInterface resultChildTags.setValue("Warning", true); resultChildTags.setValue("Alert", true); resultChildTags.setValue("Info", true); + resultChildTags.setValue("StatsWorkUnit", true); + resultChildTags.setValue("SummaryStats", true); } void appendResults(IConstWorkUnit *wu, const char *username, const char *pw) diff --git a/esp/services/common/jsonhelpers.hpp b/esp/services/common/jsonhelpers.hpp index 969dcd40783..9bedec5d2d0 100644 --- a/esp/services/common/jsonhelpers.hpp +++ b/esp/services/common/jsonhelpers.hpp @@ -329,6 +329,12 @@ namespace JsonHelpers bool log = reqTree->getPropBool("@log", false); //not in schema if (log) appendJSONValue(out, "@log", true); + bool statsToWorkunit = reqTree->getPropBool("@statsToWorkunit", false); //not in schema + if (statsToWorkunit) + appendJSONValue(out, "@statsToWorkunit", true); + bool summaryStats = reqTree->getPropBool("@summaryStats", false); //not in schema + if (summaryStats) + appendJSONValue(out, "@summaryStats", true); int tracelevel = reqTree->getPropInt("@traceLevel", -1); if (tracelevel>=0) appendJSONValue(out, "@traceLevel", tracelevel); diff --git a/esp/services/ws_ecl/ws_ecl_service.cpp b/esp/services/ws_ecl/ws_ecl_service.cpp index 12317eb08a6..c2a087e0fe8 100644 --- a/esp/services/ws_ecl/ws_ecl_service.cpp +++ b/esp/services/ws_ecl/ws_ecl_service.cpp @@ -660,6 +660,12 @@ static void buildReqXml(StringArray& parentTypes, IXmlType* type, StringBuffer& bool log = reqTree->getPropBool("@log", false); if (log) appendXMLAttr(out, "log", "true", nullptr, true); + bool statsToWorkunit = reqTree->getPropBool("@statsToWorkunit", false); + if (statsToWorkunit) + appendXMLAttr(out, "statsToWorkunit", "true", nullptr, true); + bool summaryStats = reqTree->getPropBool("@summaryStats", false); + if (summaryStats) + appendXMLAttr(out, "summaryStats", "true", nullptr, true); int tracelevel = reqTree->getPropInt("@traceLevel", -1); if (tracelevel >= 0) out.appendf(" traceLevel=\"%d\"", tracelevel); diff --git a/esp/xslt/wsecl3_form.xsl b/esp/xslt/wsecl3_form.xsl index 5382b3c9291..d2b6a7ff012 100644 --- a/esp/xslt/wsecl3_form.xsl +++ b/esp/xslt/wsecl3_form.xsl @@ -320,6 +320,14 @@ function switchInputForm() + + + +
+ Save stats to workunit.      Get summary stats.
+ + + diff --git a/esp/xslt/wsecl3_result.xslt b/esp/xslt/wsecl3_result.xslt index 9125c85c977..733fe972474 100644 --- a/esp/xslt/wsecl3_result.xslt +++ b/esp/xslt/wsecl3_result.xslt @@ -77,11 +77,29 @@ + + + + + + + + + SummaryStats: +
+
+ + + + Stats WorkUnit: +
+
+ Exception
diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index 27b10f94cd6..d659928df9e 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -1768,12 +1768,26 @@ class RoxieProtocolMsgSink : implements IHpccNativeProtocolMsgSink, public CInte Owned logwriter = protocol->writeAppendContent(nullptr); msgctx->writeLogXML(*logwriter); } - if (statsWuid.length()) + + bool summaryStats = msg->getPropBool("@summaryStats", false); + if (!statsWuid.isEmpty() || summaryStats) { Owned wuwriter = protocol->writeAppendContent(nullptr); - wuwriter->outputBeginNested("StatsWorkUnit", true); - wuwriter->outputCString(statsWuid.str(), "wuid"); - wuwriter->outputEndNested("StatsWorkUnit"); + if (!statsWuid.isEmpty()) + { + wuwriter->outputBeginNested("StatsWorkUnit", true); + wuwriter->outputCString(statsWuid.str(), "wuid"); + wuwriter->outputEndNested("StatsWorkUnit"); + } + if (summaryStats) + { + //The query completion time needs discussion and is unavailable for now. + VStringBuffer s(" COMPLETE: %s %s memory=%u Mb agentsreply=%u duplicatePackets=%u resentPackets=%u", + target, roxieMsgCtx->uid.str(), memused, agentsReplyLen, agentsDuplicates, agentsResends); + IRoxieContextLogger &logctx = static_cast(*msgctx->queryLogContext()); + logctx.getStats(s).newline(); + wuwriter->outputCString(s.str(), "SummaryStats"); + } } protocol->finalize(idx);