From e3f7710b6114df6f034e1d12922381ce77a1bc5f Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Fri, 8 Sep 2023 14:00:31 +0100 Subject: [PATCH 1/9] HPCC-30249 Release scripts for testing new indexes Signed-off-by: Gavin Halliday --- testing/regress/ecl/stresstext.ecl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testing/regress/ecl/stresstext.ecl b/testing/regress/ecl/stresstext.ecl index 3d0a896e589..7b74a662254 100644 --- a/testing/regress/ecl/stresstext.ecl +++ b/testing/regress/ecl/stresstext.ecl @@ -33,6 +33,7 @@ import ^ as root; multiPart := #IFDEFINED(root.multiPart, true); variant := #IFDEFINED(root.variant, ''); +numJoins := #IFDEFINED(root.numJoins, 40); //--- end of version configuration --- @@ -71,4 +72,4 @@ createSamples(iters, numRows) := FUNCTIONMACRO RETURN o; ENDMACRO; -createSamples(40, 60000); +createSamples(numJoins, 60000); From 582dc1aa27b2da2876d8f96b5a4c9c4e5f7840e3 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Fri, 22 Sep 2023 14:29:04 +0100 Subject: [PATCH 2/9] Split off 9.2.24 Signed-off-by: Gordon Smith --- helm/hpcc/Chart.yaml | 4 ++-- helm/hpcc/templates/_helpers.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/dali.yaml | 2 +- helm/hpcc/templates/dfuserver.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 4 ++-- helm/hpcc/templates/eclccserver.yaml | 4 ++-- helm/hpcc/templates/eclscheduler.yaml | 2 +- helm/hpcc/templates/esp.yaml | 2 +- helm/hpcc/templates/localroxie.yaml | 2 +- helm/hpcc/templates/roxie.yaml | 8 ++++---- helm/hpcc/templates/sasha.yaml | 2 +- helm/hpcc/templates/thor.yaml | 10 +++++----- version.cmake | 2 +- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/helm/hpcc/Chart.yaml b/helm/hpcc/Chart.yaml index 3ebdf68242d..08d0d40f237 100644 --- a/helm/hpcc/Chart.yaml +++ b/helm/hpcc/Chart.yaml @@ -6,9 +6,9 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 9.2.23-closedown0 +version: 9.2.25-closedown0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 9.2.23-closedown0 +appVersion: 9.2.25-closedown0 diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl index 864b4fa5522..039df424e7f 100644 --- a/helm/hpcc/templates/_helpers.tpl +++ b/helm/hpcc/templates/_helpers.tpl @@ -1378,7 +1378,7 @@ kind: Service metadata: name: {{ $lvars.serviceName | quote }} labels: - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $.root "instance" $lvars.serviceName ) | indent 4 }} {{- if $lvars.labels }} {{ toYaml $lvars.labels | indent 4 }} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 63c57a67272..2a321f1d3c2 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -50,7 +50,7 @@ spec: labels: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dafilesrv" "name" "dafilesrv" "instance" .name) | indent 8 }} server: {{ .name | quote }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 annotations: checksum/config: {{ $configSHA }} spec: diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml index 30a0997eb70..00de7016fac 100644 --- a/helm/hpcc/templates/dali.yaml +++ b/helm/hpcc/templates/dali.yaml @@ -82,7 +82,7 @@ spec: run: {{ $dali.name | quote }} server: {{ $dali.name | quote }} app: dali - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} {{- end }} diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml index e66afba9943..1f5197b2a2e 100644 --- a/helm/hpcc/templates/dfuserver.yaml +++ b/helm/hpcc/templates/dfuserver.yaml @@ -56,7 +56,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "dfuserver" "name" "dfuserver" "instance" .name) | indent 8 }} run: {{ .name | quote }} accessDali: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 02e425c200e..9cc9c60d5fe 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -58,7 +58,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" $apptype "name" "eclagent" "instance" $appJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -135,7 +135,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 410432b94f6..72262bc2b23 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -57,7 +57,7 @@ data: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclccserver" "name" "eclccserver" "instance" $compileJobName "instanceOf" (printf "%s-job" .me.name)) | indent 12 }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} {{- end }} @@ -142,7 +142,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: {{ .useChildProcesses | default false | ternary "yes" "no" | quote }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml index d9e2a4209da..7025d50de64 100644 --- a/helm/hpcc/templates/eclscheduler.yaml +++ b/helm/hpcc/templates/eclscheduler.yaml @@ -64,7 +64,7 @@ spec: run: {{ .name | quote }} accessDali: "yes" accessEsp: "no" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index c7882f93e4f..99156c418a2 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -117,7 +117,7 @@ spec: server: {{ .name | quote }} accessDali: "yes" app: {{ $application }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8 }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index 0b46ae47763..39f689fe5a5 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -70,7 +70,7 @@ spec: server: {{ $servername | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $roxie.name) | indent 8 }} {{- if hasKey . "labels" }} {{ toYaml .labels | indent 8 }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 9f6877525cf..af7d112337b 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -120,7 +120,7 @@ spec: {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 8 }} run: {{ $commonCtx.toponame | quote }} roxie-cluster: {{ $roxie.name | quote }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} @@ -180,7 +180,7 @@ kind: Service metadata: name: {{ $commonCtx.toponame | quote }} labels: - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "topology-server" "name" "roxie" "instance" $commonCtx.toponame) | indent 4 }} spec: ports: @@ -242,7 +242,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "roxie-server" "name" "roxie" "instance" $servername) | indent 8 }} {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} @@ -347,7 +347,7 @@ spec: roxie-cluster: {{ $roxie.name | quote }} accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey $.Values.global "metrics" }} {{- include "hpcc.generateMetricsReporterLabel" $.Values.global.metrics | nindent 8}} {{- end }} diff --git a/helm/hpcc/templates/sasha.yaml b/helm/hpcc/templates/sasha.yaml index c7edfeaf960..421af5dbbdc 100644 --- a/helm/hpcc/templates/sasha.yaml +++ b/helm/hpcc/templates/sasha.yaml @@ -52,7 +52,7 @@ spec: run: {{ $serviceName | quote }} server: {{ $serviceName | quote }} accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }} - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- if hasKey $sasha "labels" }} {{ toYaml $sasha.labels | indent 8 }} {{- end }} diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index 4b84caa2f1f..95e7735485f 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -82,7 +82,7 @@ data: labels: accessDali: "yes" accessEsp: "yes" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $eclAgentJobName "instanceOf" (printf "%s-job" .eclAgentName)) | indent 8 }} {{- if hasKey .me "labels" }} {{ toYaml .me.labels | indent 12 }} @@ -147,7 +147,7 @@ data: accessEsp: "yes" app: "thor" component: "thormanager" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thormanager" "name" "thor" "instance" $thorManagerJobName "instanceOf" (printf "%s-thormanager-job" .me.name)) | indent 12 }} @@ -214,7 +214,7 @@ data: accessEsp: "yes" app: "thor" component: "thorworker" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 instance: "_HPCC_JOBNAME_" job: "_HPCC_JOBNAME_" {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "thorworker" "name" "thor" "instance" $thorWorkerJobName "instanceOf" (printf "%s-thorworker-job" .me.name)) | indent 12 }} @@ -347,7 +347,7 @@ spec: accessEsp: {{ $commonCtx.eclAgentUseChildProcesses | ternary "yes" "no" | quote }} app: "thor" component: "thor-eclagent" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 instance: {{ $commonCtx.eclAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.eclAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} @@ -412,7 +412,7 @@ spec: accessEsp: "no" app: "thor" component: "thor-thoragent" - helmVersion: 9.2.23-closedown0 + helmVersion: 9.2.25-closedown0 instance: {{ $commonCtx.thorAgentName | quote }} {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "eclagent" "name" "thor" "instance" $commonCtx.thorAgentName ) | indent 8 }} {{- if hasKey $commonCtx.me "labels" }} diff --git a/version.cmake b/version.cmake index 5da52977821..5b9ced18cd4 100644 --- a/version.cmake +++ b/version.cmake @@ -5,7 +5,7 @@ set ( HPCC_NAME "Community Edition" ) set ( HPCC_PROJECT "community" ) set ( HPCC_MAJOR 9 ) set ( HPCC_MINOR 2 ) -set ( HPCC_POINT 23 ) +set ( HPCC_POINT 25 ) set ( HPCC_MATURITY "closedown" ) set ( HPCC_SEQUENCE 0 ) ### From f6d78f8d4f18210b312466138ba395cb6ccee4a0 Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Fri, 22 Sep 2023 16:24:21 -0400 Subject: [PATCH 3/9] HPCC-30312 ECL Watch v9 fix Add to Superfile dialog labels Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/forms/AddToSuperfile.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/esp/src/src-react/components/forms/AddToSuperfile.tsx b/esp/src/src-react/components/forms/AddToSuperfile.tsx index eaffb1a94ec..e026599fc8e 100644 --- a/esp/src/src-react/components/forms/AddToSuperfile.tsx +++ b/esp/src/src-react/components/forms/AddToSuperfile.tsx @@ -123,7 +123,7 @@ export const AddToSuperfile: React.FunctionComponent = ({ - + From ac9e1067002afee0ed732920ed79d09274a3a0fb Mon Sep 17 00:00:00 2001 From: Ken Rowland Date: Fri, 22 Sep 2023 16:38:47 -0400 Subject: [PATCH 4/9] HPCC-30335 Improve scope denied log message Added username and actual permission to denied log message Signed-Off-By: kenneth.rowland@lexisnexisrisk.com --- dali/base/dadfs.cpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index 713041951fe..fece27870fc 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -1379,9 +1379,23 @@ static void checkLogicalScope(const char *scopename,IUserDescriptor *user,bool r SecAccessFlags perm = getScopePermissions(scopename,user,auditflags); IDFS_Exception *e = NULL; if (readreq&&!HASREADPERMISSION(perm)) - e = new CDFS_Exception(DFSERR_LookupAccessDenied,scopename); + { + StringBuffer scopeDescription; + StringBuffer username(""); + if (user) + user->getUserName(username); + scopeDescription.appendf("%s user '%s', assigned access %s (%d)", scopename, username.str(), getSecAccessFlagName(perm), perm); + e = new CDFS_Exception(DFSERR_LookupAccessDenied,scopeDescription); + } else if (createreq&&!HASWRITEPERMISSION(perm)) - e = new CDFS_Exception(DFSERR_CreateAccessDenied,scopename); + { + StringBuffer scopeDescription; + StringBuffer username(""); + if (user) + user->getUserName(username); + scopeDescription.appendf("%s user '%s', assigned access %s (%d)", scopename, username.str(), getSecAccessFlagName(perm), perm); + e = new CDFS_Exception(DFSERR_CreateAccessDenied,scopeDescription); + } if (e) throw e; } From bda778550a8bf33e93cc5466aac62405c70c947b Mon Sep 17 00:00:00 2001 From: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> Date: Tue, 19 Sep 2023 13:42:29 -0400 Subject: [PATCH 5/9] HPCC-30276 ECL Watch v9 Super Owner not sortable update Files and WUs grids to be sortable as noted in comment in Jira Signed-off-by: Jeremy Clements <79224539+jeclrsg@users.noreply.github.com> --- esp/src/src-react/components/Files.tsx | 4 ++-- esp/src/src-react/components/Workunits.tsx | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 03f60dff097..7943e996bdf 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -175,8 +175,8 @@ export const Files: React.FunctionComponent = ({ }, }, Owner: { label: nlsHPCC.Owner }, - SuperOwners: { label: nlsHPCC.SuperOwner }, - Description: { label: nlsHPCC.Description }, + SuperOwners: { label: nlsHPCC.SuperOwner, sortable: false }, + Description: { label: nlsHPCC.Description, sortable: false }, NodeGroup: { label: nlsHPCC.Cluster }, Records: { label: nlsHPCC.Records, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 0f1e9314ed6..442569ac44d 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -144,7 +144,7 @@ export const Workunits: React.FunctionComponent = ({ Owner: { label: nlsHPCC.Owner, width: 80 }, Jobname: { label: nlsHPCC.JobName }, Cluster: { label: nlsHPCC.Cluster }, - RoxieCluster: { label: nlsHPCC.RoxieCluster }, + RoxieCluster: { label: nlsHPCC.RoxieCluster, sortable: false }, State: { label: nlsHPCC.State, width: 60 }, TotalClusterTime: { label: nlsHPCC.TotalClusterTime, width: 120, From 0ede44bb9df3bc0640de36ee0307c68fac12c85f Mon Sep 17 00:00:00 2001 From: Rodrigo Pastrana Date: Mon, 25 Sep 2023 19:10:53 -0400 Subject: [PATCH 6/9] HPCC-30352 Mute Jtrace init logging - Does not log default jtrace initialization Signed-off-by: Rodrigo Pastrana --- system/jlib/jtrace.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 278c6a117cd..87aad7cef32 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -658,8 +658,6 @@ class CTraceManager : implements ITraceManager, public CInterface DBGLOG("Tracing exporter type not specified"); } } - else - DBGLOG("Tracing exporter type 'InMemory'"); //Administrator can choose to process spans in batches or one at a time std::unique_ptr processor; @@ -756,7 +754,6 @@ class CTraceManager : implements ITraceManager, public CInterface else { initTracerProviderAndGlobalInternals(traceConfig); - DBGLOG("OpenTel tracing enabled!!"); } // The global propagator should be set regardless of whether tracing is enabled or not. From 08e4f53af1620caf7a811e05d027e75e49081bad Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 22 Aug 2023 17:22:46 +0100 Subject: [PATCH 7/9] HPCC-30140 Initial test cases for roxie https soapcall Signed-off-by: Gavin Halliday --- testing/regress/ecl/common/SoapTextTest.ecl | 132 ++++++++++++++++++ testing/regress/ecl/key/soaptext1.xml | 7 + .../ecl/setup/soaptest_getdocumentcount.ecl | 24 ++++ .../ecl/setup/soaptest_getsearchwords.ecl | 24 ++++ .../ecl/setup/soaptest_mainservice.ecl | 24 ++++ testing/regress/ecl/soaptext1.ecl | 33 +++++ 6 files changed, 244 insertions(+) create mode 100644 testing/regress/ecl/common/SoapTextTest.ecl create mode 100644 testing/regress/ecl/key/soaptext1.xml create mode 100644 testing/regress/ecl/setup/soaptest_getdocumentcount.ecl create mode 100644 testing/regress/ecl/setup/soaptest_getsearchwords.ecl create mode 100644 testing/regress/ecl/setup/soaptest_mainservice.ecl create mode 100644 testing/regress/ecl/soaptext1.ecl diff --git a/testing/regress/ecl/common/SoapTextTest.ecl b/testing/regress/ecl/common/SoapTextTest.ecl new file mode 100644 index 00000000000..afb91f99709 --- /dev/null +++ b/testing/regress/ecl/common/SoapTextTest.ecl @@ -0,0 +1,132 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//version multiPart=false,variant='default' + +import ^ as root; +multiPart := #IFDEFINED(root.multiPart, true); +variant := #IFDEFINED(root.variant, ''); + +//--- end of version configuration --- + +import $.^.setup; +files := setup.files(multiPart, false); + +EXPORT SoapTextTest := MODULE + + import Std, Setup.Ts; + EXPORT wordRec := { string word; }; + + //External service 1: How many documents does this word appear in? + EXPORT doDocumentCount(string search) := FUNCTION + cleanedSearch := Std.Str.ToLowerCase(TRIM(search)); + searchIndex := files.getSearchIndexVariant(variant); + matches := searchIndex(KEYED(kind = Ts.KindType.TextEntry AND word = cleanedSearch)) : onwarning(4523, ignore); + bydocs := TABLE(matches, { cnt := COUNT(GROUP), doc }, doc); + numDocs := COUNT(byDocs); + RETURN numDocs; + END; + + EXPORT countServiceRequest := RECORD + STRING search; + END; + EXPORT countServiceResponse := RECORD + UNSIGNED cnt; + END; + + EXPORT documentCountService() := FUNCTION + string searchWord := '' : stored('search'); + RETURN OUTPUT(ROW(TRANSFORM(countServiceResponse, SELF.cnt := doDocumentCount(searchWord)))); + END; + + //External searvice2: Given a list of words, find the documents they all occur in, and return the number of counts of each word + EXPORT joinServiceResponseRecord := RECORD + Ts.DocumentId doc; + STRING word; + UNSIGNED cnt; + END; + + EXPORT doSearchWords(DATASET(wordRec) searchWords) := FUNCTION + + searchIndex := files.getSearchIndexVariant(variant); + outputRecord := RECORDOF(searchIndex); + + doAction(SET OF DATASET(outputRecord) prev, UNSIGNED step) := FUNCTION + searchWord := searchWords[step].word; + cleanedSearch := Std.Str.ToLowerCase(TRIM(searchWord)); + matches := searchIndex(KEYED(kind = Ts.KindType.TextEntry AND word = cleanedSearch)) : onwarning(4523, ignore); + doJoin := MERGEJOIN([prev[step-1], matches], STEPPED(LEFT.doc = RIGHT.doc), SORTED(doc)); + RETURN IF (step =1, matches, doJoin); + END; + + nullInput := DATASET([], outputRecord); + results := GRAPH(nullInput, count(searchWords), doAction(ROWSET(LEFT), COUNTER), PARALLEL); + summary := TABLE(results, {doc, word, cnt := COUNT(GROUP)}, doc, word); + RETURN PROJECT(summary, TRANSFORM(joinServiceResponseRecord, SELF.word := TRIM(LEFT.word), SELF := LEFT)); + END; + + EXPORT searchWordsService() := FUNCTION + DATASET(wordRec) searchWords := DATASET([], wordRec) : stored('search'); + RETURN OUTPUT(doSearchWords(searchWords)); + END; + + EXPORT doMain(string serviceUrl, string searchWords, unsigned documentLimit) := FUNCTION + + soapcallDocumentCount(string searchWord) := SOAPCALL(serviceUrl, 'soaptest_getdocumentcount', countServiceRequest, transform(countServiceRequest, SELF.search := searchWord), countServiceResponse).cnt; + callDocumentCount(string search) := IF((serviceUrl != ''), soapcallDocumentCount(search), doDocumentCount(search)); + + soapcallSearchWords(DATASET(wordRec) searchWords) := SOAPCALL(serviceUrl, 'soaptest_getsearchwords', { DATASET(wordRec) search := searchWords }, DATASET(joinServiceResponseRecord)); + callSearchWords(DATASET(wordRec) searchWords) := IF((serviceUrl != ''), soapcallSearchWords(searchWords), doSearchWords(searchWords)); + + splitWords := Std.Str.SplitWords(searchWords, ',', false); + splitWordsDs := DATASET(splitwords, wordRec); + + wordsWithDocCounts := TABLE(splitWordsDs, { string word := word; numDocs := callDocumentCount(word); }); + // + leastCommon := TOPN(wordsWithDocCounts, documentLimit, numDocs); + + searchAgain := PROJECT(leastCommon, TRANSFORM(wordRec, SELF.word := LEFT.word)); + joinLeastCommon := callSearchWords(searchAgain); + + rollupRecord := RECORD + Ts.DocumentId doc; + SET OF STRING words; + SET OF UNSIGNED counts; + END; + + rollupRecord rollupWords(joinServiceResponseRecord l, DATASET(joinServiceResponseRecord) matches) := TRANSFORM + SELF.doc := l.doc; + sortedWords := SORT(matches, -cnt); + SELF.words := SET(sortedWords, word); + SELF.counts := SET(sortedWords, cnt); + END; + + rolledUp := ROLLUP(GROUP(joinLeastCommon, doc), GROUP, rollupWords(LEFT, ROWS(LEFT))); + RETURN rolledUp; + END; + + EXPORT mainService() := FUNCTION + // The published search service take a list of words, and a maximum number of significant documents + string searchWords := '' : stored('searchWords'); + unsigned documentLimit := 3 : stored('documentLimit'); + serviceUrl := '' : stored('url'); + unsigned maxResults := 50; + + RETURN OUTPUT(CHOOSEN(doMain(serviceUrl, searchWords, documentLimit), maxResults)); + END; + +END; diff --git a/testing/regress/ecl/key/soaptext1.xml b/testing/regress/ecl/key/soaptext1.xml new file mode 100644 index 00000000000..fe5a78da34a --- /dev/null +++ b/testing/regress/ecl/key/soaptext1.xml @@ -0,0 +1,7 @@ + + 562949953421345kingsheeprichard321 + 562949953421587kingsheeprichard321 + 562949953421614kingrichardsheep6522 + 562949953421651sheeprichardking13042 + 562949953421681kingrichardsheep211 + diff --git a/testing/regress/ecl/setup/soaptest_getdocumentcount.ecl b/testing/regress/ecl/setup/soaptest_getdocumentcount.ecl new file mode 100644 index 00000000000..6ccb6375064 --- /dev/null +++ b/testing/regress/ecl/setup/soaptest_getdocumentcount.ecl @@ -0,0 +1,24 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//nothor +//nohthor +//publish + +import common.SoapTextTest; + +SoapTextTest.documentCountService(); diff --git a/testing/regress/ecl/setup/soaptest_getsearchwords.ecl b/testing/regress/ecl/setup/soaptest_getsearchwords.ecl new file mode 100644 index 00000000000..199fb273493 --- /dev/null +++ b/testing/regress/ecl/setup/soaptest_getsearchwords.ecl @@ -0,0 +1,24 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//nothor +//nohthor +//publish + +import common.SoapTextTest; + +SoapTextTest.searchWordsService(); diff --git a/testing/regress/ecl/setup/soaptest_mainservice.ecl b/testing/regress/ecl/setup/soaptest_mainservice.ecl new file mode 100644 index 00000000000..91f1e206d7b --- /dev/null +++ b/testing/regress/ecl/setup/soaptest_mainservice.ecl @@ -0,0 +1,24 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//nothor +//nohthor +//publish + +import common.SoapTextTest; + +SoapTextTest.mainService(); diff --git a/testing/regress/ecl/soaptext1.ecl b/testing/regress/ecl/soaptext1.ecl new file mode 100644 index 00000000000..6c79d123a16 --- /dev/null +++ b/testing/regress/ecl/soaptext1.ecl @@ -0,0 +1,33 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + +//xversion url='http://.:9876' +//nothor +//nohthor + +import ^ as root; + +serviceUrl := #IFDEFINED(root.url, 'http://.:9876'); + +//--- end of version configuration --- + +import common.SoapTextTest; + +#stored ('searchWords', 'one,and,sheep,when,richard,king'); +#stored ('url', serviceUrl); + +SoapTextTest.mainService(); From 7df93fdeaaa348498be8896b1dc89ddb870ee9ad Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Tue, 26 Sep 2023 13:08:37 +0100 Subject: [PATCH 8/9] HPCC-30358 Avoid converting local hostname to a blank string Signed-off-by: Gavin Halliday --- system/jlib/jsocket.cpp | 1 - system/jlib/jstring.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/system/jlib/jsocket.cpp b/system/jlib/jsocket.cpp index f246cc8769f..420487f3afa 100644 --- a/system/jlib/jsocket.cpp +++ b/system/jlib/jsocket.cpp @@ -3598,7 +3598,6 @@ bool IpAddress::ipset(const char *text) { if (text&&*text) { if ((text[0]=='.')&&(text[1]==0)) { - hostname.set(GetCachedHostName()); // Is this better than '.'? ipset(queryHostIP()); return true; } diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp index b54d30e4838..fc8fc9c20d1 100644 --- a/system/jlib/jstring.cpp +++ b/system/jlib/jstring.cpp @@ -1434,7 +1434,7 @@ StringAttr& StringAttr::operator = (StringAttr && from) StringAttr& StringAttr::operator = (const StringAttr & from) { - set(from.str()); + set(from.get()); return *this; } From 954a2a532dcd055027602ab6c94001e614ad5ea0 Mon Sep 17 00:00:00 2001 From: Jake Smith Date: Tue, 26 Sep 2023 15:17:31 +0100 Subject: [PATCH 9/9] HPCC-30361 watchdog SocketEndpoint serialization fix Signed-off-by: Jake Smith --- thorlcr/shared/thwatchdog.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/thorlcr/shared/thwatchdog.hpp b/thorlcr/shared/thwatchdog.hpp index 2e65594d06d..ba8b5edf0be 100644 --- a/thorlcr/shared/thwatchdog.hpp +++ b/thorlcr/shared/thwatchdog.hpp @@ -30,11 +30,13 @@ struct HeartBeatPacketHeader { public: + // NB: packetSize and progressSize are back-patched and must remain at fixed offsets from the start of this struct size32_t packetSize = 0; // used as validity check must be first - SocketEndpoint sender; - unsigned tick = 0; // sequence check size32_t progressSize = 0; // size of progress data (following performance data) + unsigned tick = 0; // sequence check + SocketEndpoint sender; + public: void serialize(MemoryBuffer & out) const {
{nlsHPCC.TargetName}{nlsHPCC.LogicalName}