Merge remote-tracking branch 'origin/candidate-9.2.x' into candidate-…

…9.4.x

dali/base/dautils.cpp

@@ -124,7 +124,7 @@ bool isPathInPlane(IPropertyTree *plane, const char *path)
 
 bool validateDropZone(IPropertyTree * plane, const char * path, const char * host, bool ipMatch)
 {
-    if (host)
+    if (!isEmptyString(host))
     {
         if (!isHostInPlane(plane, host, ipMatch))
             return false;

dali/dalidiag/dalidiag.cpp

@@ -532,6 +532,8 @@ int main(int _argc, char* argv[])
     }
 
     try {
+        initNullConfiguration();
+
         Owned<IGroup> group = createIGroup(epa);
         assertex(group);
         initClientProcess(group, DCR_DaliDiag, 0, NULL, NULL, MP_WAIT_FOREVER);

dali/dalistop/dalistop.cpp

@@ -24,6 +24,13 @@
 #include "daclient.hpp"
 
 
+static void usage()
+{
+    printf("usage: dalistop <server_ip:port> [/nowait]\n");
+    printf("eg:  dalistop .                          -- stop dali server running locally\n");
+    printf("     dalistop eq0001016                  -- stop dali server running remotely\n");
+}
+
 int main(int argc, const char* argv[])
 {
     InitModuleObjects();
@@ -34,23 +41,25 @@ int main(int argc, const char* argv[])
         const char *server = nullptr;
         if (argc<2)
         {
-            // with no args, use port from daliconfig if present (used by init scripts)
-            Owned<IFile> daliConfigFile = createIFile("daliconf.xml");
-            if (daliConfigFile->exists())
-            {
-                Owned<IPropertyTree> daliConfig = createPTree(*daliConfigFile, ipt_caseInsensitive);
-                port = daliConfig->getPropInt("@port", DALI_SERVER_PORT);
-                server = ".";
-            }
+            if (isContainerized())
+                usage();
             else
             {
-                printf("usage: dalistop <server_ip:port> [/nowait]\n");
-                printf("eg:  dalistop .                          -- stop dali server running locally\n");
-                printf("     dalistop eq0001016                  -- stop dali server running remotely\n");
+                // with no args, use port from daliconfig if present (used by init scripts)
+                Owned<IFile> daliConfigFile = createIFile("daliconf.xml");
+                if (daliConfigFile->exists())
+                {
+                    Owned<IPropertyTree> daliConfig = createPTree(*daliConfigFile, ipt_caseInsensitive);
+                    port = daliConfig->getPropInt("@port", DALI_SERVER_PORT);
+                    server = ".";
+                }
+                else
+                    usage();
             }
         }
         else
         {
+            initNullConfiguration();
             server = argv[1];
             port = DALI_SERVER_PORT;
         }

dali/sasha/sasha.cpp

@@ -406,6 +406,8 @@ int main(int argc, char* argv[])
     EnableSEHtoExceptionMapping();
     Thread::setDefaultStackSize(0x10000);
     try {
+        initNullConfiguration();
+
         startMPServer(0);
         attachStandardFileLogMsgMonitor("sasha.log", NULL, MSGFIELD_STANDARD, MSGAUD_all, MSGCLS_all, TopDetail, LOGFORMAT_table, true);
         queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_prefix);

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml

@@ -214,7 +214,7 @@ myelk-kibana-68688b4d4d-d489b             1/1     Running            0      </pr
         Kibana (highlighted above) you can proceed.</para>
       </sect3>
 
-      <sect3 id="confirming_elastic_services" role="brk">
+      <sect3 id="confirming_elastic_services">
         <title>Confirming the Elastic Services</title>
 
         <para>To confirm the Elastic services are running, issue the following
@@ -279,47 +279,24 @@ myelk-kibana LoadBalancer 10.110.129.199 localhost 5601:31465/TCP 68m
         logs. The way you direct HPCC Systems to do so is by providing a
         values file that includes the log mappings. We have provided a default
         values file and we provide an example command line that inserts that
-        values file into your deployment.</para>
-
-        <para>These are the Elastic4HPCCLogs values:</para>
-
-        <programlisting># Configures HPCC logAccess to target elastic4hpcclogs deployment
-global:
-  logAccess:
-    name: "Elastic4HPCCLogs"
-    type: "elasticstack"
-    connection:
-        protocol: "http"
-        host: "elasticsearch-master.default.svc.cluster.local"
-        port: 9200
-    logMaps:
-      - type: "global"                             #These settings apply to all log mappings
-        storeName: "hpcc-logs*"                    #Logs are expected to be housed in ES inde
-        searchColumn: "message"                    #The 'message' field is to be targeted for
-        timeStampColumn: "@timestamp"              #The '@timestamp' field contains time log 
-      - type: "workunits"                          #Search by workunits specific log mapping
-        storeName: "hpcc-logs*"                    # Only needed if differs from global.store
-        searchColumn: "hpcc.log.jobid"             # Field containing WU information
-      - type: "components"                         #Search by components specific log mapping
-        searchColumn: "kubernetes.container.name"  # Field containing container information
-      - type: "audience"                           #Search by audience specific log mapping
-        searchColumn: "hpcc.log.audience"          # Field containing audience information
-      - type: "class"                              #Search by log class specific log mapping
-        searchColumn: "hpcc.log.class"             # Field containing log class information
-      - type: "instance"                           #Search by log source instance specific ma
-        searchColumn: "kubernetes.pod.name"        # Field containing source instance informa
-      - type: "host"                               #Search by log source host specific mappin
-        searchColumn: "kubernetes.node.hostname"   # Field containing source host information</programlisting>
-
-        <para>You can use the delivered Elastic4HPCCLogs chart, provided on
-        https://github.com/hpcc-systems/HPCC-Platform/tree/master/helm/managed/logging/elastic
-        or you can add the values there to your customized configuration
-        values yaml file. You can then install it using a command such
-        as:</para>
+        values file into your deployment. This suggested configuration values
+        file for enabling log access can be found in the HPCC Systems Platform
+        GitHub repository.</para>
 
-        <programlisting>helm install mycluster hpcc/hpcc -f elastic4hpcclogs-hpcc-logaccess.yaml</programlisting>
+        <para><ulink
+        url="https://github.com/hpcc-systems/HPCC-Platform">https://github.com/hpcc-systems/HPCC-Platform</ulink></para>
+
+        <para>Then navigating to the
+        <emphasis>helm/examples/azure/log-analytics/loganalytics-hpcc-logaccess.yaml</emphasis>
+        file.</para>
 
-        <para></para>
+        <para>You can use the delivered Elastic4HPCCLogs chart provided or you
+        can add the values there to your customized configuration values yaml
+        file.</para>
+
+        <para>You can then install it using a command such as:</para>
+
+        <programlisting>helm install mycluster hpcc/hpcc -f elastic4hpcclogs-hpcc-logaccess.yaml</programlisting>
       </sect3>
     </sect2>
   </sect1>
@@ -619,10 +596,10 @@ ContainerLog
       <title>Target Audience Filtering</title>
 
       <para>The availble target audiences include operator(OPR), user(USR),
-      programmer(PRO), monitor(MON), audit(ADT), or all. The filter is controlled by the
-      &lt;section&gt;.logging.audiences value. The string value is comprised
-      of 3 letter codes delimited by the aggregation operator (+) or the
-      removal operator (-).</para>
+      programmer(PRO), monitor(MON), audit(ADT), or all. The filter is
+      controlled by the &lt;section&gt;.logging.audiences value. The string
+      value is comprised of 3 letter codes delimited by the aggregation
+      operator (+) or the removal operator (-).</para>
 
       <para>For example, all component log output to include Programmer and
       User messages only:</para>
@@ -634,8 +611,8 @@ ContainerLog
       <title>Target Category Filtering</title>
 
       <para>The available target categories include disaster(DIS), error(ERR),
-      information(INF), warning(WRN), progress(PRO), event(EVT), metrics(MET). The
-      category (or class) filter is controlled by the
+      information(INF), warning(WRN), progress(PRO), event(EVT), metrics(MET).
+      The category (or class) filter is controlled by the
       &lt;section&gt;.logging.classes value, comprised of 3 letter codes
       delimited by the aggregation operator (+) or the removal operator
       (-).</para>

esp/src/eclwatch/ECLPlaygroundWidget.js

@@ -337,8 +337,9 @@ define([
                 var context = this;
                 hpccComms.Workunit.compile({ baseUrl: "" }, selectedTarget.Name, ecl).then(function (wu) {
                     context.wu = wu;
+                    context.wu.update({ Jobname: jobname });
                     return context.wu.watchUntilComplete(function () {
-                        context.updateInput("State", "", wu.State);
+                        context.updateInput("State", "", context.wu.State);
                     });
                 }).then(function () {
                     context.updateInput("State", "", "Publishing");

esp/src/src-react/components/GroupMembers.tsx

@@ -58,7 +58,7 @@ export const GroupMembers: React.FunctionComponent<GroupMembersProps> = ({
     const columns = React.useMemo((): FluentColumns => {
         return {
             username: {
-                label: nlsHPCC.UserName,
+                label: nlsHPCC.Username,
                 formatter: (_name, idx) => {
                     return <Link href={`#/${opsCategory}/security/users/${_name}`}>{_name}</Link>;
                 }

esp/src/src-react/components/PackageMaps.tsx

@@ -367,8 +367,8 @@ export const PackageMaps: React.FunctionComponent<PackageMapsProps> = ({
                 const _targets: TypedDropdownOption[] = [{ key: "*", text: "ANY" }];
                 const _processes: TypedDropdownOption[] = [{ key: "*", text: "ANY" }];
                 Targets?.TargetData.map(target => {
-                    if (_targets.filter(t => t.key === target.Type).length === 0) {
-                        _targets.push({ key: target.Type, text: target.Type, type: target.Type });
+                    if (_targets.filter(t => t.key === target.Name).length === 0) {
+                        _targets.push({ key: target.Name, text: target.Name, type: target.Type });
                     }
                     target?.Processes?.Item.map(item => {
                         if (_processes.filter(p => p.key === item).length === 0) {

esp/src/src-react/components/Search.tsx

@@ -29,6 +29,9 @@ const searchResultUrl = (result) => {
         case "Logical File":
             url = "#/files/" + result._nodeGroup + "/" + result._name;
             break;
+        case "Super File":
+            url = "#/files/" + result._name;
+            break;
         case "Query":
             url = "#/queries/" + result._querySetId + "/" + result._id;
             break;
@@ -112,7 +115,7 @@ export const Search: React.FunctionComponent<SearchProps> = ({
             key: "open", text: nlsHPCC.Open, disabled: !uiState.hasSelection, iconProps: { iconName: "WindowEdit" },
             onClick: () => {
                 if (selection.length === 1) {
-                    window.location.href = `#/workunits/${selection[0].Wuid}`;
+                    window.location.href = searchResultUrl(selection[0]);
                 } else {
                     for (let i = selection.length - 1; i >= 0; --i) {
                         window.open(searchResultUrl(selection[i]), "_blank");

esp/src/src-react/components/forms/GroupAddUser.tsx

@@ -81,7 +81,7 @@ export const GroupAddUserForm: React.FunctionComponent<GroupAddUserProps> = ({
                     key={fieldName}
                     groupname={groupname}
                     required={true}
-                    label={nlsHPCC.GroupName}
+                    label={nlsHPCC.Username}
                     selectedKey={value}
                     onChange={(evt, option) => {
                         onChange(option.key);

esp/src/src-react/components/forms/landing-zone/FileListForm.tsx

@@ -80,17 +80,23 @@ export const FileListForm: React.FunctionComponent<FileListFormProps> = ({
                 .then(response => response.json())
                 .then(response => {
                     setSubmitDisabled(false);
-                    const DFUActionResult = response?.UploadFilesResponse?.UploadFileResults?.DFUActionResult;
+                    const exceptions = response?.Exceptions?.Exception ?? [];
+                    if (exceptions.length > 0) {
+                        logger.error(exceptions[0]?.Message ?? nlsHPCC.ErrorUploadingFile);
+                        return;
+                    }
+                    const DFUActionResult = response?.UploadFilesResponse?.UploadFileResults?.DFUActionResult ?? [];
                     if (DFUActionResult.filter(result => result.Result !== "Success").length > 0) {
-                        console.log("upload failed");
+                        logger.error(nlsHPCC.ErrorUploadingFile);
                     } else {
                         closeForm();
                         if (typeof onSubmit === "function") {
                             onSubmit();
                         }
                         reset(defaultValues);
                     }
-                });
+                })
+                .catch(err => logger.error(err));
         };
 
         handleSubmit(

esp/src/src/Session.ts

@@ -48,13 +48,15 @@ getBuildInfo().then(info => {
     dojoConfig.currencyCode = info["currencyCode"] ?? "";
 });
 
-const format = d3Format(",.6f");
+const formatTwoDigits = d3Format(",.2f");
+const formatSixDigits = d3Format(",.6f");
 export function formatCost(value): string {
     if (isNaN(value)) {
         logger.debug(`formatCost called for a nullish value: ${value}`);
         return "";
     }
     const _number = typeof value === "string" ? Number(value) : value;
+    const format = (_number > 0 && _number < 1) ? formatSixDigits : formatTwoDigits;
     return `${format(_number)} (${dojoConfig?.currencyCode || "USD"})`;
 }
 

helm/hpcc/templates/dali.yaml

@@ -101,6 +101,7 @@ spec:
     spec:
       {{- include "hpcc.placementsByPodTargetType" (dict "root" $ "pod" $dali.name "type" "dali") | indent 6 }}
       serviceAccountName: "hpcc-dali"
+      terminationGracePeriodSeconds: {{ .terminationGracePeriodSeconds | default 3600 }}
       initContainers: 
       {{- include "hpcc.createConfigInitContainers" (dict "root" $ "me" $dali "includeCategories" $tmpDaliScope.aggregatePlaneCategories "includeNames" $tmpDaliScope.aggregateSashaNamedPlanes) | indent 6 }}
       {{- include "hpcc.addImagePullSecrets" $commonCtx | nindent 6 -}}

helm/hpcc/templates/esp.yaml

@@ -116,6 +116,9 @@ spec:
         run: {{ .name | quote }}
         server: {{ .name | quote }}
         accessDali: "yes"
+{{- if eq .application "eclwatch" }}
+        accessSasha: "yes"
+{{- end }}
         app: {{ $application }}
         helmVersion: 9.4.5-closedown0
         {{- include "hpcc.addStandardLabels" (dict "root" $ "name" $application "component" "esp" "instance" .name) | indent 8 }}

helm/hpcc/templates/network.yaml

@@ -157,6 +157,21 @@ spec:
         matchLabels:
           app: eclservices
 ---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: sasha-server
+spec:
+  podSelector:
+    matchLabels:
+      app: sasha
+  ingress:
+  - from:
+    - podSelector:
+        matchLabels:
+          accessSasha: "yes"
+---
+
 ##
 ## Provides access to pods labeled prometheusMetricsReporter: "yes"
 ## onto port declared in global.metrics.sinks[type=prometheus].configuration.port

helm/hpcc/templates/sasha.yaml

@@ -51,6 +51,7 @@ spec:
         {{- include "hpcc.addStandardLabels" (dict "root" $ "component" "sasha" "name" "sasha" "instance" $serviceName) | indent 8 }}
         run: {{ $serviceName | quote }}
         server: {{ $serviceName | quote }}
+        app: sasha
         accessDali: {{ (has "dali" $sasha.access) | ternary "yes" "no" | quote }}
         helmVersion: 9.4.5-closedown0
 {{- if hasKey $sasha "labels" }}

helm/hpcc/values.schema.json

@@ -1292,6 +1292,10 @@
         },
         "egress": {
           "$ref" : "#/definitions/egress"
+        },
+        "terminationGracePeriodSeconds": {
+          "$ref": "#/definitions/terminationGracePeriodSeconds",
+          "default": 3600
         }
       }
     },
@@ -1366,6 +1370,9 @@
           "type": "string",
           "description": "The default repo version used if not supplied for the defaultRepo"
         },
+        "terminationGracePeriodSeconds": {
+          "$ref": "#/definitions/terminationGracePeriodSeconds"
+        },
         "resources": {
           "$ref": "#/definitions/resources"
         },
@@ -1411,6 +1418,9 @@
         },
         "resources": {
           "$ref": "#/definitions/resources"
+        },
+        "terminationGracePeriodSeconds": {
+          "$ref": "#/definitions/terminationGracePeriodSeconds"
         }
       }
     },
@@ -3276,6 +3286,12 @@
         }
       },
       "required": [ "name", "type", "target" ]
+    },
+    "terminationGracePeriodSeconds": {
+      "type": "integer",
+      "description": "Period permitted for component to terminate gracefully before being killed by Kubernetes",
+      "default": 600,
+      "minimum": 0
     }
   }
 }

initfiles/sbin/hpcc_setenv.in

@@ -114,15 +114,14 @@ umask ${OUMASK}
 
 # use less heap when threaded
 # but if set too low it can affect performance significantly
-num_arenas=$num_cpus
-if [ -z "$num_arenas" ] ; then
-    num_arenas=8
-elif [ $num_arenas -lt 8 ] ; then
-    num_arenas=8
-elif [ $num_arenas -gt 32 ] ; then
-    num_arenas=32
+if [ -z "$MALLOC_ARENA_MAX" ]; then
+    if [ $num_cpus -gt 8 ]; then
+        num_arenas=8
+    else
+        num_arenas=$num_cpus
+    fi
+    export MALLOC_ARENA_MAX=$num_arenas
 fi
-export MALLOC_ARENA_MAX=$num_arenas
 
 PATH_PREFIX=`cat ${HPCC_CONFIG} | sed -n "/\[${SECTION}\]/,/\[/p" | grep "^path *= *" | sed -e 's/^path *= *//'`