Merge remote-tracking branch 'origin/candidate-9.6.x'

Signed-off-by: Jake Smith <[email protected]>

.github/actions/hpcc4j-esp-regression-suite/action.yaml

@@ -38,17 +38,20 @@ runs:
         github_token: ${{ inputs.github-token }}
         branch_name: ${{ inputs.branch-name }}
         repository: ${{ inputs.repository }}
+        actions_run_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
       shell: python
       run: |
         import os
         import csv
         import textwrap
         import json
+        import sys
 
         branch_name = os.getenv('branch_name')
         comments_url = os.getenv('comments_url')
         github_token  = os.getenv('github_token')
         repository = os.getenv('repository')
+        actions_run_url = os.getenv('actions_run_url')
 
         file_path = "./HPCC4j/wsclient/FailedTests.csv"
         if os.path.exists(file_path):
@@ -60,17 +63,19 @@ runs:
               testFailureMessages += textwrap.dedent(f"""\
               ## {row[0]}.{row[1]} Failed
               **Error:** ```{row[2]}```
-              **Test Command:** ```mvn -B -Dhpccconn=https://eclwatch.default:8010 -Dtest={row[0]}#{row[1]} test```
+              **Test Command:** ```mvn -B --activate-profiles jenkins-on-demand -pl wsclient -Dhpccconn=https://eclwatch.default:8010 -Dtest={row[0]}#{row[1]} test```
               """)
 
             if testFailureMessages:
               message = textwrap.dedent(f"""\
               # ESP Regression Suite Test Failures:
+              **Action Run:** {actions_run_url}
               The following tests failed; checkout a copy of the HPCC4j project with the following command and run the individual test commands below to debug the failures.
               ```
               git clone https://github.com/{repository}.git hpcc4j
               cd hpcc4j && git checkout {branch_name}
               ```
+              **See:** https://github.com/hpcc-systems/hpcc4j/wiki/Running-Individual-Unit-Tests for detailed information on running the tests below.
               """)
               message += testFailureMessages
 

dali/base/dasds.cpp

@@ -5527,7 +5527,7 @@ class CStoreHelper : implements IStoreHelper, public CInterface
                                 if (newName.charAt(i)==':')
                                     newName.setCharAt(i,'_');
                             newName.append(".unused");
-                            PROGLOG("Detected spurious data file : '%s' - renaming to %s", file.queryFilename(), newName.str());
+                            PROGLOG("Detected spurious data file : '%s' - renaming to %s (current edition=%u, file edition=%u)", file.queryFilename(), newName.str(), edition, fileEdition);
                             try
                             {
                                 file.rename(newName.str());

dali/server/daldap.cpp

@@ -54,6 +54,7 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
     Owned<ISecManager>      ldapsecurity;
     StringAttr              filesdefaultuser;
     StringAttr              filesdefaultpassword;
+    bool                    disableFilesDefaultUser;
     unsigned                ldapflags;
     IDigitalSignatureManager * pDSM = nullptr;
 
@@ -82,6 +83,7 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
             {
                 filesdefaultuser.set(ldapprops->queryProp("@filesDefaultUser"));
                 filesdefaultpassword.set(ldapprops->queryProp("@filesDefaultPassword"));
+                disableFilesDefaultUser = ldapprops->getPropBool("@disableDefaultUser", false);
 
                 try {
                     ignoreSigPipe(); // LDAP can generate
@@ -115,29 +117,36 @@ class CDaliLdapConnection: implements IDaliLdapConnection, public CInterface
             return SecAccess_Full;
 
 
+        Owned<ISecUser> user;
         StringBuffer username;
         StringBuffer password;
         if (udesc)
         {
             udesc->getUserName(username);
             udesc->getPassword(password);
+            user.setown(ldapsecurity->createUser(username));
+            user->setAuthenticateStatus(AS_AUTHENTICATED);  // treat caller passing user as trusted
         }
         else
         {
             DBGLOG("NULL UserDescriptor in daldap.cpp getPermissions('%s')", key);
-        }
+            logNullUser(nullptr);
+
+            // If no user was provided, try to use the default user
+            if (disableFilesDefaultUser || filesdefaultuser.isEmpty())
+            {
+                OWARNLOG("Default user missing or disabled, access denied for request %s %s", key, nullText(obj));
+                return SecAccess_None; // no access if no default user or disabled
+            }
 
-        if (0 == username.length())
-        {
             username.append(filesdefaultuser);
             decrypt(password, filesdefaultpassword);
-            OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser for request %s %s", key, nullText(obj));
-            logNullUser(nullptr);
+            OWARNLOG("Missing credentials, injecting deprecated filesdefaultuser (%s) for request %s %s", filesdefaultuser.str(), key,
+                     nullText(obj));
+            user.setown(ldapsecurity->createUser(username));
+            user->credentials().setPassword(password);  // Force authentication of default user when used
         }
 
-        Owned<ISecUser> user = ldapsecurity->createUser(username);
-        user->setAuthenticateStatus(AS_AUTHENTICATED);
-
         SecAccessFlags perm = SecAccess_None;
         unsigned start = msTick();
         if (filescope)

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml

@@ -108,6 +108,29 @@
       The next sections will show you how to install and set up the Elastic
       stack logging solution for HPCC Systems.</para>
 
+      <para><informaltable colsep="1" frame="all" rowsep="1">
+          <?dbfo keep-together="always"?>
+
+          <tgroup cols="2">
+            <colspec colwidth="49.50pt" />
+
+            <colspec />
+
+            <tbody>
+              <row>
+                <entry><inlinegraphic
+                fileref="../../images/caution.png" /></entry>
+
+                <entry><emphasis role="bold">NOTE: </emphasis>The
+                elastic4hpcclogs chart does not enable any security. The
+                responsibility of determining the need for security and
+                enabling security on any deployed Elastic Stack instance or
+                components is up to you and your organization. </entry>
+              </row>
+            </tbody>
+          </tgroup>
+        </informaltable></para>
+
       <sect3 id="logs_Add_theHPCC_Systems_Repo">
         <title>Add the HPCC Systems Repository</title>
 

ecl/eclccserver/eclccserver.cpp

@@ -778,7 +778,7 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter,
                 }
                 else
                 {
-                    if (processKilled)
+                    if (processKilled && !workunit->aborting())
                         addExceptionToWorkunit(workunit, SeverityError, "eclccserver", 9999, "eclcc killed - likely to be out of memory - see compile log for details", nullptr, 0, 0, 0);
 #ifndef _CONTAINERIZED
                     Owned<IWUQuery> query = workunit->updateQuery();

ecllibrary/std/Date.ecl

@@ -1647,6 +1647,7 @@ EXPORT TZ_DATA := DATASET
             {'CHAST', 45900, ['PACIFIC']}, // Chatham Island Standard Time
             {'CHOST', 32400, ['ASIA']}, // Choibalsan Summer Time
             {'CHOT', 28800, ['ASIA']}, // Choibalsan Time
+            {'CHST', 36000, ['PACIFIC']}, // Chamorro Standard Time
             {'ChST', 36000, ['PACIFIC']}, // Chamorro Standard Time
             {'CHUT', 36000, ['PACIFIC']}, // Chuuk Time
             {'CIDST', -14400, ['CARIBBEAN']}, // Cayman Islands Daylight Saving Time

esp/clients/ws_dfsclient/ws_dfsclient.cpp

@@ -862,6 +862,35 @@ IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, Acc
     return lookup(lfn, user, accessMode, hold, lockSuperOwner, transaction, priviledged, timeout);
 }
 
+bool exists(CDfsLogicalFileName &lfn, IUserDescriptor *user, bool notSuper, bool superOnly, unsigned timeout)
+{
+    if (!lfn.isRemote())
+        return queryDistributedFileDirectory().exists(lfn.get(), user, notSuper, superOnly);
+
+    Owned<IDistributedFile> file = lookup(lfn, user, AccessMode::read, false, false, nullptr, false, timeout);
+    if (!file)
+        return false;
+    bool isSuper = nullptr != file->querySuperFile();
+    if (superOnly)
+    {
+        if (!isSuper)
+            return false;
+    }
+    else if (notSuper)
+    {
+        if (isSuper)
+            return false;
+    }
+    return true;
+}
+
+bool exists(const char *logicalFilename, IUserDescriptor *user, bool notSuper, bool superOnly, unsigned timeout)
+{
+    CDfsLogicalFileName lfn;
+    lfn.set(logicalFilename);
+    return exists(lfn, user, notSuper, superOnly, timeout);
+}
+
 
 } // namespace wsdfs
 

esp/clients/ws_dfsclient/ws_dfsclient.hpp

@@ -53,6 +53,8 @@ WS_DFSCLIENT_API IDistributedFile *lookupLegacyDFSFile(const char *logicalName,
 
 WS_DFSCLIENT_API IDistributedFile *lookup(CDfsLogicalFileName &lfn, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);
 WS_DFSCLIENT_API IDistributedFile *lookup(const char *logicalFilename, IUserDescriptor *user, AccessMode accessMode, bool hold, bool lockSuperOwner, IDistributedFileTransaction *transaction, bool priviledged, unsigned timeout);
+WS_DFSCLIENT_API bool exists(CDfsLogicalFileName &lfn, IUserDescriptor *user, bool notSuper, bool superonly, unsigned timeout);
+WS_DFSCLIENT_API bool exists(const char *logicalFilename, IUserDescriptor *user, bool notWuper, bool superOnly, unsigned timeout);
 
 
 } // end of namespace wsdfs