Merge remote-tracking branch 'origin/candidate-9.4.x' into candidate-…

…9.6.x

Signed-off-by: Jake Smith <[email protected]>

.github/actions/hpcc4j-esp-regression-suite/action.yaml

@@ -38,17 +38,20 @@ runs:
         github_token: ${{ inputs.github-token }}
         branch_name: ${{ inputs.branch-name }}
         repository: ${{ inputs.repository }}
+        actions_run_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
       shell: python
       run: |
         import os
         import csv
         import textwrap
         import json
+        import sys
 
         branch_name = os.getenv('branch_name')
         comments_url = os.getenv('comments_url')
         github_token  = os.getenv('github_token')
         repository = os.getenv('repository')
+        actions_run_url = os.getenv('actions_run_url')
 
         file_path = "./HPCC4j/wsclient/FailedTests.csv"
         if os.path.exists(file_path):
@@ -60,17 +63,19 @@ runs:
               testFailureMessages += textwrap.dedent(f"""\
               ## {row[0]}.{row[1]} Failed
               **Error:** ```{row[2]}```
-              **Test Command:** ```mvn -B -Dhpccconn=https://eclwatch.default:8010 -Dtest={row[0]}#{row[1]} test```
+              **Test Command:** ```mvn -B --activate-profiles jenkins-on-demand -pl wsclient -Dhpccconn=https://eclwatch.default:8010 -Dtest={row[0]}#{row[1]} test```
               """)
 
             if testFailureMessages:
               message = textwrap.dedent(f"""\
               # ESP Regression Suite Test Failures:
+              **Action Run:** {actions_run_url}
               The following tests failed; checkout a copy of the HPCC4j project with the following command and run the individual test commands below to debug the failures.
               ```
               git clone https://github.com/{repository}.git hpcc4j
               cd hpcc4j && git checkout {branch_name}
               ```
+              **See:** https://github.com/hpcc-systems/hpcc4j/wiki/Running-Individual-Unit-Tests for detailed information on running the tests below.
               """)
               message += testFailureMessages
 

dali/base/dasds.cpp

@@ -5527,7 +5527,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                                 if (newName.charAt(i)==':')
                                     newName.setCharAt(i,'_');
                             newName.append(".unused");
-                            PROGLOG("Detected spurious data file : '%s' - renaming to %s", file.queryFilename(), newName.str());
+                            PROGLOG("Detected spurious data file : '%s' - renaming to %s (current edition=%u, file edition=%u)", file.queryFilename(), newName.str(), edition, fileEdition);
                             try
                             {
                                 file.rename(newName.str());

ecl/eclccserver/eclccserver.cpp

@@ -778,7 +778,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 }
                 else
                 {
-                    if (processKilled)
+                    if (processKilled && !workunit->aborting())
                         addExceptionToWorkunit(workunit, SeverityError, "eclccserver", 9999, "eclcc killed - likely to be out of memory - see compile log for details", nullptr, 0, 0, 0);
 #ifndef _CONTAINERIZED
                     Owned<IWUQuery> query = workunit->updateQuery();

esp/clients/ws_dfsclient/ws_dfsclient.cpp

@@ -862,6 +862,35 @@ IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, Acc
     return lookup(lfn, user, accessMode, hold, lockSuperOwner, transaction, priviledged, timeout);
 }
 
+bool exists(CDfsLogicalFileName &lfn, IUserDescriptor *user, bool notSuper, bool superOnly, unsigned timeout)
+{
+    if (!lfn.isRemote())
+        return queryDistributedFileDirectory().exists(lfn.get(), user, notSuper, superOnly);
+
+    Owned<IDistributedFile> file = lookup(lfn, user, AccessMode::read, false, false, nullptr, false, timeout);
+    if (!file)
+        return false;
+    bool isSuper = nullptr != file->querySuperFile();
+    if (superOnly)
+    {
+        if (!isSuper)
+            return false;
+    }
+    else if (notSuper)
+    {
+        if (isSuper)
+            return false;
+    }
+    return true;
+}
+
+bool exists(const char *logicalFilename, IUserDescriptor *user, bool notSuper, bool superOnly, unsigned timeout)
+{
+    CDfsLogicalFileName lfn;
+    lfn.set(logicalFilename);
+    return exists(lfn, user, notSuper, superOnly, timeout);
+}
+
 
 } // namespace wsdfs
 

esp/clients/ws_dfsclient/ws_dfsclient.hpp

@@ -53,6 +53,8 @@ WS_DFSCLIENT_API IDistributedFile *lookupLegacyDFSFile(const char *logicalName,
 
 WS_DFSCLIENT_API IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);
 WS_DFSCLIENT_API IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);
+WS_DFSCLIENT_API bool exists(CDfsLogicalFileName &lfn, IUserDescriptor *user, bool notSuper, bool superonly, unsigned timeout);
+WS_DFSCLIENT_API bool exists(const char *logicalFilename, IUserDescriptor *user, bool notWuper, bool superOnly, unsigned timeout);
 
 
 } // end of namespace wsdfs

esp/src/src-react/components/DataPatterns.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, mergeStyleSets, ScrollablePane, ScrollbarVisibility, Sticky, StickyPositionType } from "@fluentui/react";
+import { CommandBar, ContextualMenuItemType, ICommandBarItemProps, IDropdownOption, mergeStyleSets, ScrollablePane, ScrollbarVisibility, Sticky, StickyPositionType } from "@fluentui/react";
 import nlsHPCC from "src/nlsHPCC";
 import { DPWorkunit } from "src/DataPatterns/DPWorkunit";
 import { Report } from "src/DataPatterns/Report";
@@ -87,7 +87,7 @@ export const DataPatterns: React.FunctionComponent<DataPatternsProps> = ({
                 className={dpStyles.inlineDropdown}
                 required={true}
                 selectedKey={targetCluster}
-                onChange={(ev, row) => {
+                onChange={(ev, row: IDropdownOption) => {
                     setTargetCluster(row.key as string);
                 }}
             />

esp/src/src-react/components/Workunits.tsx

@@ -25,7 +25,7 @@ const FilterFields: Fields = {
     "Wuid": { type: "string", label: nlsHPCC.WUID, placeholder: "W20200824-060035" },
     "Owner": { type: "string", label: nlsHPCC.Owner, placeholder: nlsHPCC.jsmi },
     "Jobname": { type: "string", label: nlsHPCC.JobName, placeholder: nlsHPCC.log_analysis_1 },
-    "Cluster": { type: "target-cluster", label: nlsHPCC.Cluster, placeholder: "" },
+    "Cluster": { type: "target-cluster", label: nlsHPCC.Cluster, placeholder: "", multiSelect: true },
     "State": { type: "workunit-state", label: nlsHPCC.State, placeholder: "" },
     "ECL": { type: "string", label: nlsHPCC.ECL, placeholder: nlsHPCC.dataset },
     "LogicalFile": { type: "string", label: nlsHPCC.LogicalFile, placeholder: nlsHPCC.somefile },

esp/src/src-react/components/forms/AddBinding.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { DefaultButton, PrimaryButton, TextField, } from "@fluentui/react";
+import { DefaultButton, IDropdownOption, PrimaryButton, TextField, } from "@fluentui/react";
 import { scopedLogger } from "@hpcc-js/util";
 import { useForm, Controller } from "react-hook-form";
 import { EsdlDefinitionsTextField, EsdlEspProcessesTextField } from "./Fields";
@@ -76,7 +76,7 @@ export const AddBindingForm: React.FunctionComponent<AddBindingFormProps> = ({
                 fieldState: { error }
             }) => <EsdlEspProcessesTextField
                     key={fieldName}
-                    onChange={(evt, option) => {
+                    onChange={(evt, option: IDropdownOption) => {
                         onChange(option.key);
                     }}
                     required={true}
@@ -108,7 +108,7 @@ export const AddBindingForm: React.FunctionComponent<AddBindingFormProps> = ({
                 fieldState: { error }
             }) => <EsdlDefinitionsTextField
                     key={fieldName}
-                    onChange={(evt, option) => {
+                    onChange={(evt, option: IDropdownOption) => {
                         onChange(option.key);
                     }}
                     required={true}

esp/src/src-react/components/forms/AddPermission.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { DefaultButton, MessageBar, MessageBarType, PrimaryButton, TextField, } from "@fluentui/react";
+import { DefaultButton, IDropdownOption, MessageBar, MessageBarType, PrimaryButton, TextField, } from "@fluentui/react";
 import { scopedLogger } from "@hpcc-js/util";
 import { useForm, Controller } from "react-hook-form";
 import nlsHPCC from "src/nlsHPCC";
@@ -82,7 +82,7 @@ export const AddPermissionForm: React.FunctionComponent<AddPermissionFormProps>
                     required={true}
                     label={nlsHPCC.Type}
                     selectedKey={value}
-                    onChange={(evt, option) => {
+                    onChange={(evt, option: IDropdownOption) => {
                         onChange(option.key);
                     }}
                     errorMessage={error && error?.message}

esp/src/src-react/components/forms/CopyFile.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { Checkbox, DefaultButton, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react";
+import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react";
 import { useForm, Controller } from "react-hook-form";
 import nlsHPCC from "src/nlsHPCC";
 import * as FileSpray from "src/FileSpray";
@@ -130,7 +130,7 @@ export const CopyFile: React.FunctionComponent<CopyFileProps> = ({
                         required={true}
                         selectedKey={value}
                         placeholder={nlsHPCC.SelectValue}
-                        onChange={(evt, option) => {
+                        onChange={(evt, option: IDropdownOption) => {
                             onChange(option.key);
                         }}
                         errorMessage={error && error.message}

esp/src/src-react/components/forms/DesprayFile.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { Checkbox, DefaultButton, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react";
+import { Checkbox, DefaultButton, IDropdownOption, mergeStyleSets, PrimaryButton, Stack, TextField, } from "@fluentui/react";
 import { useForm, Controller } from "react-hook-form";
 import { FileSpray, FileSprayService } from "@hpcc-js/comms";
 import { scopedLogger } from "@hpcc-js/util";
@@ -151,7 +151,7 @@ export const DesprayFile: React.FunctionComponent<DesprayFileProps> = ({
                         required={true}
                         selectedKey={value}
                         placeholder={nlsHPCC.SelectValue}
-                        onChange={(evt, option) => {
+                        onChange={(evt, option: IDropdownOption) => {
                             setDropzone(option.key as string);
                             setDirectory(option["path"] as string);
                             if (option["path"].indexOf("\\") > -1) {
@@ -179,7 +179,7 @@ export const DesprayFile: React.FunctionComponent<DesprayFileProps> = ({
                             label={nlsHPCC.IPAddress}
                             selectedKey={value}
                             placeholder={nlsHPCC.SelectValue}
-                            onChange={(evt, option) => {
+                            onChange={(evt, option: IDropdownOption) => {
                                 setMachine(option.key as string);
                                 setOs(option["OS"] as number);
                                 onChange(option.key);