Merge remote-tracking branch 'origin/candidate-9.4.x'

Signed-off-by: Gordon Smith <[email protected]>

common/wuwebview/wuwebview.cpp

@@ -80,6 +80,8 @@ class WuExpandedResultBuffer : implements IPTreeNotifyEvent, public CInterface
         resultChildTags.setValue("Warning", true);
         resultChildTags.setValue("Alert", true);
         resultChildTags.setValue("Info", true);
+        resultChildTags.setValue("StatsWorkUnit", true);
+        resultChildTags.setValue("SummaryStats", true);
     }
 
     void appendResults(IConstWorkUnit *wu, const char *username, const char *pw)

dali/base/dafdesc.cpp

@@ -35,6 +35,8 @@
 #include "jsecrets.hpp"
 #include "rmtfile.hpp"
 
+#include <memory>
+
 #define INCLUDE_1_OF_1    // whether to use 1_of_1 for single part files
 
 #define SDS_CONNECT_TIMEOUT  (1000*60*60*2)     // better than infinite
@@ -944,35 +946,38 @@ void getClusterInfo(IPropertyTree &pt, INamedGroupStore *resolver, unsigned flag
                 cgroup.setown(resolver->lookup(grp));
             // get nodes from parts if complete (and group 0)
             if (gi==0) { // don't assume lookup name correct!
-                SocketEndpoint *eps = (SocketEndpoint *)calloc(np?np:1,sizeof(SocketEndpoint));
                 MemoryBuffer mb;
                 Owned<IPropertyTreeIterator> piter;
                 if (pt.getPropBin("Parts",mb))
                     piter.setown(deserializePartAttrIterator(mb));
                 else
                     piter.setown(pt.getElements("Part"));
+
                 ForEach(*piter) {
                     IPropertyTree &cpt = piter->query();
                     unsigned num = cpt.getPropInt("@num");
-                    if (num>np) {
-                        eps = (SocketEndpoint *)checked_realloc(eps,num*sizeof(SocketEndpoint),np*sizeof(SocketEndpoint),-21);
-                        memset(eps+np,0,(num-np)*sizeof(SocketEndpoint));
+                    if (num>np)
                         np = num;
-                    }
+                }
+
+                std::unique_ptr<SocketEndpoint[]> eps(new SocketEndpoint[np?np:1]);
+                ForEach(*piter) {
+                    IPropertyTree &cpt = piter->query();
+                    unsigned num = cpt.getPropInt("@num");
                     const char *node = cpt.queryProp("@node");
                     if (node&&*node)
-                        eps[num-1].set(node);
+                        eps.get()[num-1].set(node);
                 }
                 unsigned i=0;
                 for (i=0;i<np;i++)
                     if (eps[i].isNull())
                         break;
                 if (i==np) {
-                    Owned<IGroup> ngrp = createIGroup(np,eps);
+                    Owned<IGroup> ngrp = createIGroup(np,eps.get());
                     if (!cgroup.get()||(ngrp->compare(cgroup)!=GRbasesubset))
                         cgroup.setown(ngrp.getClear());
                 }
-                free(eps);
+
             }
             ClusterPartDiskMapSpec mspec;
             IClusterInfo *cluster = createClusterInfo(grp,cgroup,mspec,resolver);

docs/EN_US/ECLLanguageReference/ECLR_mods/BltInFunc-SOAPCALL.xml

@@ -69,7 +69,7 @@
             form usernames and passwords, if required. If calling an ESP Web
             service, you can append the ver_=n.nn parameter to specify the
             version of the service. For example: <programlisting
-            language="ECL" role="notrunnable">SOAPCALL('http://127.0.0.1:8010/Wsdfu/?ver_=1.22', 
+            language="ECL" role="notrunnable">SOAPCALL('https://eclwatch.example.com:8010/Wsdfu/?ver_=1.22', 
          'DFUSearchData', 
          instructure,DATASET(outsructure));</programlisting></entry>
           </row>
@@ -444,10 +444,18 @@
   STRING500 OutData{XPATH('OutData')};
   UNSIGNED4 Latency{XPATH('_call_latency')};
 END;
-ip := 'http://127.0.0.1:8022/';
-ips := 'https://127.0.0.1:8022/';
-ipspw := 'https://username:[email protected]:8022/';
+ip := 'http://service.example.com:8022/';
+ips := 'https://service.example.com:8022/';
 svc := 'MyModule.SomeService';
+ips_secret := 'secret:myConnectSecret';
+        /* assumes a secret named http-connect-myConnectSecret exists &amp; contains:
+        /*
+         {
+          "url": "https://service.example.com:8022/",
+          "username": "username",
+          "password": "password"
+         }
+        */ 
 
 //1 rec in, 1 rec out
 OneRec1 := SOAPCALL(ips,svc,{STRING500 InData := 'Some Input Data'},OutRec1);
@@ -460,7 +468,7 @@ OneRec2 := SOAPCALL(InputDataset,ip,svc,{STRING500 InData},OutRec1);
 
 //recordset in, recordset out
 ManyRec2 := 
- SOAPCALL(InputDataset,ipspw,svc,{STRING500 InData := 'Some In Data'},DATASET(OutRec1));
+ SOAPCALL(InputDataset,ips_secret,svc,{STRING500 InData := 'Some In Data'},DATASET(OutRec1));
 
 //TRANSFORM function usage example
 namesRecord := RECORD
@@ -494,20 +502,17 @@ outRecord genDefault2(namesRecord l) := TRANSFORM
   SELF.score := 0;
 END;
 
-ip := 'http://127.0.0.1:8022/';
-svc:= 'MyModule.SomeService';
 OUTPUT(SOAPCALL(ip, svc,{ STRING20 surname := 'Halligan',STRING20 forename := 'Kevin';},
+
 DATASET(outRecord), ONFAIL(genDefault1())));
 
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(genDefault2(LEFT))));
-
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP)));
 
 //Using HTTPHEADER to pass Authorization info
 OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP),
                 HTTPHEADER('Authorization','Basic dXNlcm5hbWU6cGFzc3dvcmQ='),
                 HTTPHEADER('MyLiteral','FOO')));
-
 </programlisting>
   </sect2>
 
@@ -524,11 +529,11 @@ OUTPUT(SOAPCALL(ds, ip, svc, inRecord, t(LEFT),DATASET(outRecord), ONFAIL(SKIP),
     <para>Example:</para>
 
     <programlisting lang="ECL" role="notrunnable">//1 rec in, no result
-SOAPCALL( 'https://127.0.0.1:8022/','MyModule.SomeService',
+SOAPCALL( 'https://service.example.com:8022/','MyModule.SomeService',
            {STRING500 InData := 'Some Input Data'});
 
 //recordset in, no result
-SOAPCALL( InputDataset,'https://127.0.0.1:8022/','MyModule.SomeService',{STRING500 InData});
+SOAPCALL( InputDataset,'https://service.example.com:8022/','MyModule.SomeService',{STRING500 InData});
 </programlisting>
 
     <para>See Also: <link linkend="RECORD_Structure">RECORD Structure</link>,

docs/EN_US/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -278,15 +278,18 @@
         <sect3 id="Installing_CentOS-RedHat-Suse">
           <title>CentOS/Red Hat</title>
 
-          <para>To install the Platform you should have the appropriate
-          permissions to install packages. If you have sudo rights, then you
-          can install the platform using yum.</para>
+          <para>To install the HPCC Systems Platform you should have the
+          appropriate rights and permissions to install packages on your
+          system.</para>
+
+          <para>One way to install the platform is using yum.</para>
 
-          <programlisting>sudo yum install &lt;hpccsystems platform rpm package&gt;</programlisting>
+          <programlisting lang="bash">sudo yum install -y epel-release 
+sudo yum install -y &lt;hpccsystems platform rpm package&gt;</programlisting>
 
-          <para>Optionally you can install the package with rpm (recommended
-          using the -Uvh options), however then you would have to negotiate
-          installing any additional dependencies.</para>
+          <para>Optionally you can install packages with rpm (recommended
+          using the -Uvh options), however you would then have to negotiate
+          installing any dependencies.</para>
         </sect3>
 
         <sect3 id="Installing_Ubuntu-Debian">

ecl/eclagent/eclagent.cpp

@@ -3128,8 +3128,11 @@ char * EclAgent::getClusterName()
 char * EclAgent::getGroupName()
 {
 #ifdef _CONTAINERIZED
-    // in a containerized setup, the group is moving..
-    return strdup("unknown");
+    // In containerized there is no associated cluster group.
+    // Return an empty result instead, which if used for destinationGroup
+    // via Thorlib.group() as part of a fileservices call, will be interpreted
+    // as targeting the default data plane
+    return strdup("");
 #else
     StringBuffer groupName;
     if (!isStandAloneExe)

esp/bindings/SOAP/Platform/soapbind.cpp

@@ -271,24 +271,37 @@ int CHttpSoapBinding::HandleSoapRequest(CHttpRequest* request, CHttpResponse* re
     return 0;
 }
 
-
-static IPropertyTree *createSecClientConfig(const char *clientCertPath, const char *clientPrivateKey, const char *caCertsPath, bool acceptSelfSigned)
+static IPropertyTree *createSecClientConfig(const char *clientCertFileOrBuf, const char *clientPrivKeyFileOrBuf, const char *caCertsPathOrBuf, bool acceptSelfSigned)
 {
     Owned<IPropertyTree> info = createPTree();
 
-    if (!isEmptyString(clientCertPath))
+    if (!isEmptyString(clientCertFileOrBuf))
     {
-        info->setProp("certificate", clientCertPath);
-        if (!isEmptyString(clientPrivateKey))
-            info->setProp("privatekey", clientPrivateKey);
+        if (containsEmbeddedKey(clientCertFileOrBuf))
+            info->setProp("certificate_pem", clientCertFileOrBuf);
+        else
+            info->setProp("certificate", clientCertFileOrBuf);
+
+        if (!isEmptyString(clientPrivKeyFileOrBuf))
+        {
+            if (containsEmbeddedKey(clientPrivKeyFileOrBuf))
+                info->setProp("privatekey_pem", clientPrivKeyFileOrBuf);
+            else
+                info->setProp("privatekey", clientPrivKeyFileOrBuf);
+        }
     }
 
     IPropertyTree *verify = ensurePTree(info, "verify");
-    if (!isEmptyString(caCertsPath))
+
+    if (!isEmptyString(caCertsPathOrBuf))
     {
         IPropertyTree *ca = ensurePTree(verify, "ca_certificates");
-        ca->setProp("@path", caCertsPath);
+        if (containsEmbeddedKey(caCertsPathOrBuf))
+            ca->setProp("pem", caCertsPathOrBuf);
+        else
+            ca->setProp("@path", caCertsPathOrBuf);
     }
+
     verify->setPropBool("@enable", true);
     verify->setPropBool("@accept_selfsigned", acceptSelfSigned);
     verify->setProp("trusted_peers", "anyone");
@@ -313,6 +326,7 @@ void CSoapRequestBinding::post(const char *proxy, const char* url, IRpcResponseB
         soapclient.setReadTimeoutSecs(readTimeoutSecs_);
     if (mtls_secret_.length())
         soapclient.setMtlsSecretName(mtls_secret_);
+
     if (client_cert_.length() || ca_certs_.length() || accept_self_signed_)
         soapclient.setSecureSocketConfig(createSecClientConfig(client_cert_, client_priv_key_, ca_certs_, accept_self_signed_));
 

esp/bindings/SOAP/Platform/soapbind.hpp

@@ -181,12 +181,17 @@ class esp_http_decl CSoapRequestBinding : public CSoapComplexType,
     void setMtlsSecretName(const char *name){mtls_secret_.set(name);}
     const char *getMtlsSecretName(){return mtls_secret_.str();}
 
-    void setCACertificates(const char *path) override {ca_certs_.set(path);}
+    void setCACertificates(const char *path) override
+    {
+        ca_certs_.set(path);
+    }
+
     virtual void setClientCertificate(const char *certPath, const char *privateKeyPath) override
     {
         client_cert_.set(certPath);
         client_priv_key_.set(privateKeyPath);
     }
+
     virtual void setAcceptSelfSigned(bool acceptSelfSigned) override {accept_self_signed_=acceptSelfSigned;}
 
     void post(const char *proxy, const char* url, IRpcResponseBinding& response, const char *action=NULL);
@@ -208,20 +213,25 @@ inline void setRpcSSLOptions(IEspClientRpcSettings &rpc, bool useSSL, const char
     {
         if (!isEmptyString(clientCert))
         {
-            if (!checkFileExists(clientCert))
-                throw makeStringExceptionV(-1,"Client certificate not found %s.", clientCert);
             if (isEmptyString(clientPrivateKey))
                 throw makeStringException(-1,"Client private key not provided.");
-            if (!checkFileExists(clientPrivateKey))
+
+            if (!containsEmbeddedKey(clientCert) && !checkFileExists(clientCert))
+                throw makeStringExceptionV(-1,"Client certificate not found %s.", clientCert);
+            if (!containsEmbeddedKey(clientPrivateKey) && !checkFileExists(clientPrivateKey))
                 throw makeStringExceptionV(-1,"Client private key not found %s.", clientPrivateKey);
+
             rpc.setClientCertificate(clientCert, clientPrivateKey);
         }
+
         if (!isEmptyString(caCert))
         {
-            if (!checkFileExists(caCert))
+            if (!containsEmbeddedKey(caCert) && !checkFileExists(caCert))
                 throw makeStringExceptionV(-1,"CA certificate not found %s.", caCert);
+
             rpc.setCACertificates(caCert);
         }
+
         rpc.setAcceptSelfSigned(acceptSelfSigned);
     }
 }

esp/bindings/http/platform/httpbinding.cpp

@@ -309,7 +309,6 @@ EspHttpBinding::EspHttpBinding(IPropertyTree* tree, const char *bindname, const
     Owned<IPropertyTree> bnd_cfg = getBindingConfig(tree, bindname, procname);
     m_wsdlVer=0.0;
     processName.set(procname);
-    getServiceXmlFilename(serviceXmlFilename);
 
     // get the config default version
     const char* defVersion = bnd_cfg->queryProp("@defaultServiceVersion");

esp/bindings/http/platform/httpbinding.hpp

@@ -111,7 +111,6 @@ interface IEspWsdlSections
 //  virtual MethodInfoArray & queryQualifiedNames(IEspContext& ctx)=0;
     virtual int getQualifiedNames(IEspContext& ctx, MethodInfoArray & methods)=0;
     virtual int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0;
-    virtual int getServiceXmlFilename(StringBuffer &filename)=0;
     virtual int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0;
     virtual int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0;
     virtual int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda)=0;
@@ -192,7 +191,6 @@ class esp_http_decl EspHttpBinding :
 
     Owned<IEspCorsHelper> corsHelper;
 
-    StringBuffer            serviceXmlFilename;
 
     void getXMLMessageTag(IEspContext& ctx, bool isRequest, const char *method, StringBuffer& tag);
 
@@ -202,6 +200,7 @@ class esp_http_decl EspHttpBinding :
     bool                    m_includeJsonTest;
     StringBuffer            m_challenge_realm;
     StringAttr              m_defaultSvcVersion;
+    StringBuffer            serviceXmlFilename;
 
 public:
     EspHttpBinding(IPropertyTree* cfg, const char *bindname=NULL, const char *procname=NULL);
@@ -347,7 +346,6 @@ class esp_http_decl EspHttpBinding :
 //  MethodInfoArray &queryQualifiedNames(IEspContext& ctx) { m_methods.popAll(); getQualifiedNames(ctx,m_methods); return m_methods;};
 
     int getXsdDefinition(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda){return 0;};
-    int getServiceXmlFilename(StringBuffer &filename) {return 0;};
     int getWsdlMessages(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda);
     int getWsdlPorts(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda);
     int getWsdlBindings(IEspContext &context, CHttpRequest *request, StringBuffer &content, const char *service, const char *method, bool mda);