diff --git a/common/dllserver/dllserver.cpp b/common/dllserver/dllserver.cpp index 5b2524455e5..028ee42c324 100644 --- a/common/dllserver/dllserver.cpp +++ b/common/dllserver/dllserver.cpp @@ -503,7 +503,7 @@ void DllServer::doRegisterDll(const char * name, const char * kind, const char * RemoteFilename dllRemote; StringBuffer ipText, dllText; dllRemote.setRemotePath(dllPath); - dllRemote.queryIP().getIpText(ipText); + dllRemote.queryIP().getHostText(ipText); dllRemote.getLocalPath(dllText); Owned conn = getEntryConnection(name, RTM_LOCK_WRITE); diff --git a/common/environment/dalienv.cpp b/common/environment/dalienv.cpp index fdd491760e7..4d17343e1b5 100644 --- a/common/environment/dalienv.cpp +++ b/common/environment/dalienv.cpp @@ -113,7 +113,7 @@ EnvMachineOS queryOS(const IpAddress & ip) Owned factory = getEnvironmentFactory(true); Owned env = factory->openEnvironment(); StringBuffer ipText; - ip.getIpText(ipText); + ip.getHostText(ipText); Owned machine = env->getMachineByAddress(ipText.str()); if (machine) ret = machine->getOS(); @@ -226,7 +226,7 @@ const char * querySlaveExecutable(const char * keyName, const char * exeName, co Owned factory = getEnvironmentFactory(true); Owned env = factory->openEnvironment(); StringBuffer addr; - ip.getIpText(addr); + ip.getHostText(addr); StringBufferAdaptor spp(progpath); StringBufferAdaptor swd(workdir); @@ -264,10 +264,10 @@ bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * v // use dafilesrv to work out OS StringBuffer dalis; if (remotedali) - remotedali->endpoint().getUrlStr(dalis); + remotedali->endpoint().getEndpointHostText(dalis); // first get machine by IP StringBuffer ips; - ip.getIpText(ips); + ip.getHostText(ips); //Cannot use getEnvironmentFactory() since it is using a remotedali StringBuffer xpath; diff --git a/common/environment/environment.cpp b/common/environment/environment.cpp index 21bca55d53b..d3d33cf2a2e 100644 --- a/common/environment/environment.cpp +++ b/common/environment/environment.cpp @@ -1972,7 +1972,7 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByAddressPath(const char * ne #ifdef _DEBUG StringBuffer serverIpString; - serverIP.getIpText(serverIpString); + serverIP.getHostText(serverIpString); LOG(MCdebugInfo, unknownJob, "Listed server: '%s', IP: '%s'", dropzoneServer.str(), serverIpString.str()); #endif if (strisame(netaddress, dropzoneServer) || targetIp.ipequals(serverIP)) diff --git a/common/pkgfiles/referencedfilelist.cpp b/common/pkgfiles/referencedfilelist.cpp index 552d00b6ef8..6a2d4b035fc 100644 --- a/common/pkgfiles/referencedfilelist.cpp +++ b/common/pkgfiles/referencedfilelist.cpp @@ -423,7 +423,7 @@ IPropertyTree *ReferencedFile::getSpecifiedOrRemoteFileTree(IUserDescriptor *use if (!fileTree) return NULL; StringAttrBuilder daliipText(daliip); - remote->endpoint().getUrlStr(daliipText); + remote->endpoint().getEndpointHostText(daliipText); filePrefix.set(remotePrefix); return fileTree.getClear(); } @@ -471,7 +471,7 @@ void ReferencedFile::resolveRemote(IUserDescriptor *user, INode *remote, const c flags |= RefFileNotFound; StringBuffer dest; - DBGLOG("Remote ReferencedFile not found %s [dali=%s, remote=%s, prefix=%s]", logicalName.str(), daliip.get(), remote ? remote->endpoint().getUrlStr(dest).str() : nullptr, remotePrefix); + DBGLOG("Remote ReferencedFile not found %s [dali=%s, remote=%s, prefix=%s]", logicalName.str(), daliip.get(), remote ? remote->endpoint().getEndpointHostText(dest).str() : nullptr, remotePrefix); } void ReferencedFile::resolve(const StringArray &locations, const char *srcCluster, IUserDescriptor *user, INode *remote, const char *remotePrefix, bool checkLocalFirst, StringArray *subfiles, bool _trackSubFiles, bool resolveForeign) @@ -1025,7 +1025,7 @@ void ReferencedFileList::cloneRelationships() return; StringBuffer addr; - remote->endpoint().getUrlStr(addr); + remote->endpoint().getEndpointHostText(addr); IDistributedFileDirectory &dir = queryDistributedFileDirectory(); ReferencedFileIterator files(this); ForEach(files) diff --git a/common/remote/rmtspawn.cpp b/common/remote/rmtspawn.cpp index f8bab0ea7be..64f213b1fcb 100644 --- a/common/remote/rmtspawn.cpp +++ b/common/remote/rmtspawn.cpp @@ -111,7 +111,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint unsigned port = SLAVE_CONNECT_PORT + ((unsigned)kind * NUM_SLAVE_CONNECT_PORT) + getRandom() % NUM_SLAVE_CONNECT_PORT; StringBuffer args; - myEP.getUrlStr(args); + myEP.getEndpointHostText(args); args.append(' ').append(replyTag).append(' ').append((unsigned)kind).append(" ").append(port); if (extra) args.append(' ').append(extra); @@ -177,7 +177,7 @@ ISocket *spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint try { StringBuffer tmp; - connectEP.getUrlStr(tmp); + connectEP.getEndpointHostText(tmp); LOG(MCdetailDebugInfo, unknownJob, "Try to connect to slave %s",tmp.str()); Owned socket = ISocket::connect_wait(connectEP,MASTER_CONNECT_SLAVE_TIMEOUT); if (socket) @@ -288,7 +288,7 @@ bool CRemoteParentInfo::processCommandLine(int argc, const char * * argv, String void CRemoteParentInfo::log() { StringBuffer temp; - LOG(MCdebugProgress, unknownJob, "Starting remote slave. Master=%s reply=%d port=%d", parent.getUrlStr(temp).str(), replyTag, port); + LOG(MCdebugProgress, unknownJob, "Starting remote slave. Master=%s reply=%d port=%d", parent.getEndpointHostText(temp).str(), replyTag, port); } bool CRemoteParentInfo::sendReply(unsigned version) @@ -331,7 +331,7 @@ bool CRemoteParentInfo::sendReply(unsigned version) if (version == connectVersion) { buffer.read(connectTag); - masterIP.getIpText(masterIPtext.clear()); + masterIP.getHostText(masterIPtext.clear()); LOG(MCdetailDebugInfo, unknownJob, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str()); diff --git a/common/remote/rmtssh.cpp b/common/remote/rmtssh.cpp index 9621167b9d4..86682580c7e 100644 --- a/common/remote/rmtssh.cpp +++ b/common/remote/rmtssh.cpp @@ -136,7 +136,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH if (sa.ordinality()) { StringBuffer ns; ForEachItemIn(i,sa) { - sa.item(i).getIpText(ns.clear()); + sa.item(i).getHostText(ns.clear()); slaves.append(ns.str()); } return; @@ -548,7 +548,7 @@ class CFRunSSH: public CInterface, implements IFRunSSH strict = false; verbose = false; StringBuffer ips; - ip.getIpText(ips); + ip.getHostText(ips); slaves.kill(); slaves.append(ips.str()); numthreads = 1; diff --git a/common/roxiecommlib/roxiecommunicationclient.cpp b/common/roxiecommlib/roxiecommunicationclient.cpp index 5806ef03f05..37cefbafc38 100644 --- a/common/roxiecommlib/roxiecommunicationclient.cpp +++ b/common/roxiecommlib/roxiecommunicationclient.cpp @@ -232,7 +232,7 @@ class CRoxieCommunicationClient: implements IRoxieCommunicationClient, public CI if (remoteIP) buf.append(remoteIP); else - ep.getUrlStr(buf); + ep.getEndpointHostText(buf); throw MakeStringException(ROXIECOMM_SOCKET_ERROR, "%s", buf.str()); } diff --git a/common/thorhelper/persistent.cpp b/common/thorhelper/persistent.cpp index ff738608fcf..52133b0d2d0 100644 --- a/common/thorhelper/persistent.cpp +++ b/common/thorhelper/persistent.cpp @@ -56,7 +56,7 @@ class CPersistentInfo : implements IInterface, public CInterface { if(_ep) { - _ep->getUrlStr(epstr); + _ep->getEndpointHostText(epstr); keystr.set(epstr); addKeySuffix(proto, keystr); } @@ -121,7 +121,7 @@ class CAvailKeeper private: inline StringBuffer& calcKey(SocketEndpoint& ep, PersistentProtocol proto, StringBuffer& keystr) { - ep.getUrlStr(keystr); + ep.getEndpointHostText(keystr); return addKeySuffix(proto, keystr); } SocketSet* findSet(CPersistentInfo* info, bool create = false) @@ -214,7 +214,7 @@ class CPersistentHandler : implements IPersistentHandler, implements ISocketSele if (m_enableDoNotReuseList && ep != nullptr) { StringBuffer epstr; - ep->getUrlStr(epstr); + ep->getEndpointHostText(epstr); if(m_doNotReuseList.getValue(epstr.str()) != nullptr) { PERSILOG(PersistentLogLevel::PLogNormal, "PERSISTENT: socket %d's target endpoint %s is in DoNotReuseList, will not add it.", sock->OShandle(), epstr.str()); @@ -434,7 +434,7 @@ class CPersistentHandler : implements IPersistentHandler, implements ISocketSele if(!ep) return false; StringBuffer epstr; - ep->getUrlStr(epstr); + ep->getEndpointHostText(epstr); if(epstr.length()> 0 && m_doNotReuseList.getValue(epstr.str()) != nullptr) return true; return false; diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 085f0e18d91..28f99263a69 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -106,6 +106,11 @@ class Url : public CInterface, implements IInterface return url.append(method).append("://").append(host).append(":").append(port).append(path); } + StringBuffer &getDynamicUrlSecretName(StringBuffer &secretName) const + { + return generateDynamicUrlSecretName(secretName, method, userPasswordPair, host, port, path); + } + IException *getUrlException(IException *e) const { StringBuffer url; @@ -340,7 +345,7 @@ class BlackLister : public CInterface, implements IThreadFactory if (soapTraceLevel > 3) { StringBuffer s; - logctx.CTXLOG("endpoint %s is blacklisted", ep.getUrlStr(s).str()); + logctx.CTXLOG("endpoint %s is blacklisted", ep.getEndpointHostText(s).str()); } return true; } @@ -364,7 +369,7 @@ class BlackLister : public CInterface, implements IThreadFactory if (soapTraceLevel > 0) { StringBuffer s; - logctx.CTXLOG("Blacklisting endpoint %s", p.ep.getUrlStr(s).str()); + logctx.CTXLOG("Blacklisting endpoint %s", p.ep.getEndpointHostText(s).str()); } } else @@ -372,7 +377,7 @@ class BlackLister : public CInterface, implements IThreadFactory if (soapTraceLevel > 3) { StringBuffer s; - logctx.CTXLOG("Endpoint %s is already blacklisted", p.ep.getUrlStr(s).str()); + logctx.CTXLOG("Endpoint %s is already blacklisted", p.ep.getEndpointHostText(s).str()); } } @@ -390,7 +395,7 @@ class BlackLister : public CInterface, implements IThreadFactory if ((match != NotFound) && (soapTraceLevel > 0)) { StringBuffer s; - DBGLOG("De-blacklisting endpoint %s", ep.getUrlStr(s).str()); + DBGLOG("De-blacklisting endpoint %s", ep.getEndpointHostText(s).str()); } } @@ -413,7 +418,7 @@ class BlackLister : public CInterface, implements IThreadFactory if (useBlacklister && lookup(ep, logctx)) { StringBuffer s; - ep.getUrlStr(s); + ep.getEndpointHostText(s); throw MakeStringException(JSOCKERR_connection_failed, "%s %s", blOptions->getBLerror(), s.str()); } Owned exc; @@ -470,7 +475,7 @@ class BlackLister : public CInterface, implements IThreadFactory } if (exc->errorCode()==JSOCKERR_connection_failed) { StringBuffer s; - ep.getUrlStr(s); + ep.getEndpointHostText(s); throw MakeStringException(JSOCKERR_connection_failed, "connection failed %s", s.str()); } throw exc.getClear(); @@ -552,23 +557,31 @@ class BlackLister : public CInterface, implements IThreadFactory } *blacklist; static IPersistentHandler* persistentHandler = nullptr; -static CriticalSection persistentCrit; -static std::atomic persistentInitDone{false}; +static CriticalSection globalFeatureCrit; +static std::atomic globalFeaturesInitDone{false}; +static std::atomic mapUrlsToSecrets{false}; +static std::atomic warnIfUrlNotMappedToSecret{false}; +static std::atomic requireUrlsMappedToSecrets{false}; -void initPersistentHandler() +void initGlobalFeatures() { - CriticalBlock block(persistentCrit); - if (!persistentInitDone) + CriticalBlock block(globalFeatureCrit); + if (!globalFeaturesInitDone) { -#ifndef _CONTAINERIZED - int maxPersistentRequests = queryEnvironmentConf().getPropInt("maxHttpCallPersistentRequests", 0); -#else + int maxPersistentRequests = 0; + if (!isContainerized()) + maxPersistentRequests = queryEnvironmentConf().getPropInt("maxHttpCallPersistentRequests", maxPersistentRequests); //global (backward compatible) + Owned conf = getComponentConfig(); - int maxPersistentRequests = conf->getPropInt("@maxHttpCallPersistentRequests", 0); -#endif + maxPersistentRequests = conf->getPropInt("@maxHttpCallPersistentRequests", maxPersistentRequests); //component config wins + mapUrlsToSecrets = conf->getPropBool("@mapHttpCallUrlsToSecrets", false); + warnIfUrlNotMappedToSecret = conf->getPropBool("@warnIfUrlNotMappedToSecret", mapUrlsToSecrets); + requireUrlsMappedToSecrets = conf->getPropBool("@requireUrlsMappedToSecrets", false); + if (maxPersistentRequests != 0) persistentHandler = createPersistentHandler(nullptr, DEFAULT_MAX_PERSISTENT_IDLE_TIME, maxPersistentRequests, PersistentLogLevel::PLogMin, true); - persistentInitDone = true; + + globalFeaturesInitDone = true; } } @@ -707,6 +720,8 @@ IColumnProvider * CreateColumnProvider(unsigned _callLatencyMs, bool _encoding) enum WSCType{STsoap, SThttp} ; //web service call type +static const char * getWsCallTypeName(WSCType wscType) { return wscType == STsoap ? "SOAPCALL" : "HTTPCALL"; } + //Web Services Call Asynchronous For interface IWSCAsyncFor: public IInterface { @@ -861,6 +876,41 @@ class CWSCHelperThread : public Thread } }; +bool loadConnectSecret(const char *vaultId, const char *secretName, UrlArray &urlArray, StringBuffer &issuer, StringBuffer &proxyAddress, bool required, WSCType wscType) +{ + Owned secret; + if (!isEmptyString(secretName)) + secret.setown(getSecret("ecl", secretName, vaultId, nullptr)); + if (!secret) + { + if (required) + throw MakeStringException(0, "%s %s SECRET not found", getWsCallTypeName(wscType), secretName); + return false; + } + + StringBuffer url; + getSecretKeyValue(url, secret, "url"); + if (url.isEmpty()) + throw MakeStringException(0, "%s %s HTTP SECRET must contain url", getWsCallTypeName(wscType), secretName); + UrlListParser urlListParser(url); + StringBuffer usernamePasswordPair; + getSecretKeyValue(usernamePasswordPair, secret, "username"); + if (usernamePasswordPair.length()) + { + if (strchr(usernamePasswordPair, ':')) + throw MakeStringException(0, "%s HTTP-CONNECT SECRET username contains illegal colon", getWsCallTypeName(wscType)); + StringBuffer password; + getSecretKeyValue(password, secret, "password"); + if (password.length()) + usernamePasswordPair.append(':').append(password); + } + urlListParser.getUrls(urlArray, usernamePasswordPair); + getSecretKeyValue(proxyAddress.clear(), secret, "proxy"); + getSecretKeyValue(issuer, secret, "issuer"); + return true; +} + + //================================================================================================= class CWSCHelper : implements IWSCHelper, public CInterface @@ -876,6 +926,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface static CriticalSection secureContextCrit; static Owned tlsSecureContext; static Owned localMtlsSecureContext; + static Owned remoteMtlsSecureContext; Owned customSecureContext; @@ -883,8 +934,9 @@ class CWSCHelper : implements IWSCHelper, public CInterface bool complete; std::atomic_bool timeLimitExceeded{false}; bool customClientCert = false; - bool localClientCert = false; + StringAttr clientCertIssuer; IRoxieAbortMonitor * roxieAbortMonitor; + StringBuffer issuer; //TBD sync up with other PR, it will benefit from this being able to come from the secret protected: IArrayOf threads; @@ -1018,19 +1070,24 @@ class CWSCHelper : implements IWSCHelper, public CInterface const char *hosts = hostsString.get(); if (isEmptyString(hosts)) - throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s specified no URLs", getWsCallTypeName(wscType)); if (0==strncmp(hosts, "mtls:", 5)) { - localClientCert = true; + clientCertIssuer.set("local"); hosts += 5; } + else if (0==strncmp(hosts, "remote-mtls:", 12)) + { + clientCertIssuer.set("remote"); + hosts += 12; + } if (0==strncmp(hosts, "secret:", 7)) { const char *finger = hosts+7; if (isEmptyString(finger)) - throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET specified with no name", wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s HTTP-CONNECT SECRET specified with no name", getWsCallTypeName(wscType)); if (!proxyAddress.isEmpty()) - throw MakeStringException(0, "%sCALL PROXYADDRESS can't be used with HTTP-CONNECT secrets", wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s PROXYADDRESS can't be used with HTTP-CONNECT secrets", getWsCallTypeName(wscType)); StringAttr vaultId; const char *thumb = strchr(finger, ':'); if (thumb) @@ -1040,43 +1097,45 @@ class CWSCHelper : implements IWSCHelper, public CInterface } StringBuffer secretName("http-connect-"); secretName.append(finger); - Owned secret = (vaultId.isEmpty()) ? getSecret("ecl", secretName) : getVaultSecret("ecl", vaultId, secretName, nullptr); - if (!secret) - throw MakeStringException(0, "%sCALL %s SECRET not found", wscType == STsoap ? "SOAP" : "HTTP", secretName.str()); - - StringBuffer url; - getSecretKeyValue(url, secret, "url"); - if (url.isEmpty()) - throw MakeStringException(0, "%sCALL %s HTTP SECRET must contain url", wscType == STsoap ? "SOAP" : "HTTP", secretName.str()); - UrlListParser urlListParser(url); - StringBuffer auth; - getSecretKeyValue(auth, secret, "username"); - if (auth.length()) - { - if (strchr(auth, ':')) - throw MakeStringException(0, "%sCALL HTTP-CONNECT SECRET username contains illegal colon", wscType == STsoap ? "SOAP" : "HTTP"); - auth.append(':'); - getSecretKeyValue(auth, secret, "password"); - } - urlListParser.getUrls(urlArray, auth); - proxyAddress.set(secret->queryProp("proxy")); - getSecretKeyValue(proxyAddress.clear(), secret, "proxy"); + loadConnectSecret(vaultId, secretName, urlArray, issuer, proxyAddress, true, wscType); } else { UrlListParser urlListParser(hosts); urlListParser.getUrls(urlArray); + if (mapUrlsToSecrets && urlArray.length()) + { + StringBuffer secretName; + UrlArray tempArray; + //TBD: If this is a list of URLs do we A. not check for a mapped secret, B. check the first one, C. Use long secret name including entire list + Url &url = urlArray.tos(); + url.getDynamicUrlSecretName(secretName); + if (secretName.length()) + { + if (loadConnectSecret(nullptr, secretName, tempArray, issuer, proxyAddress, requireUrlsMappedToSecrets, wscType)) + { + logctx.CTXLOG("Mapped %s URL!", wscCallTypeText()); + if (tempArray.length()) + urlArray.swapWith(tempArray); + } + else if (warnIfUrlNotMappedToSecret) + { + //should we warn even if the url doesn't have credentials embedded? If HTTPHEADER is being used to pass credentials, we still prefer connect secrets be used instead. + logctx.CTXLOG("Security Warning: %s not using a connection secret (auto secret = %s)", wscCallTypeText(), secretName.str()); + } + } + } } numUrls = urlArray.ordinality(); if (numUrls == 0) - throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s specified no URLs", getWsCallTypeName(wscType)); if (!proxyAddress.isEmpty()) { UrlListParser proxyUrlListParser(proxyAddress); if (0 == proxyUrlListParser.getUrls(proxyUrlArray)) - throw MakeStringException(0, "%sCALL proxy address specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); + throw MakeStringException(0, "%s proxy address specified no URLs", getWsCallTypeName(wscType)); } if (wscMode == SCrow) @@ -1184,8 +1243,8 @@ class CWSCHelper : implements IWSCHelper, public CInterface { if (clientCert != NULL) ownedSC.setown(createSecureSocketContextEx(clientCert->certificate, clientCert->privateKey, clientCert->passphrase, ClientSocket)); - else if (localClientCert) - ownedSC.setown(createSecureSocketContextSecret("local", ClientSocket)); + else if (clientCertIssuer.length()) + ownedSC.setown(createSecureSocketContextSecret(clientCertIssuer.str(), ClientSocket)); else ownedSC.setown(createSecureSocketContext(ClientSocket)); } @@ -1194,8 +1253,13 @@ class CWSCHelper : implements IWSCHelper, public CInterface ISecureSocketContext *ensureStaticSecureContext() { CriticalBlock b(secureContextCrit); - if (localClientCert) - return ensureSecureContext(localMtlsSecureContext); + if (clientCertIssuer.length()) + { + if (strieq(clientCertIssuer.str(), "local")) + return ensureSecureContext(localMtlsSecureContext); + if (strieq(clientCertIssuer.str(), "remote")) + return ensureSecureContext(remoteMtlsSecureContext); + } return ensureSecureContext(tlsSecureContext); } ISecureSocket *createSecureSocket(ISocket *sock, const char *fqdn = nullptr) @@ -1248,7 +1312,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface } } inline IXmlToRowTransformer * getRowTransformer() { return rowTransformer; } - inline const char * wscCallTypeText() const { return wscType == STsoap ? "SOAPCALL" : "HTTPCALL"; } + inline const char * wscCallTypeText() const { return getWsCallTypeName(wscType); } protected: friend class CWSCHelperThread; @@ -1333,6 +1397,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface CriticalSection CWSCHelper::secureContextCrit; Owned CWSCHelper::tlsSecureContext; // created on first use Owned CWSCHelper::localMtlsSecureContext; // created on first use +Owned CWSCHelper::remoteMtlsSecureContext; // created on first use //================================================================================================= @@ -1594,11 +1659,15 @@ int CWSCHelperThread::run() IWSCHelper * createSoapCallHelper(IWSCRowProvider *r, IEngineRowAllocator * outputAllocator, const char *authToken, WSCMode wscMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor) { + if (!globalFeaturesInitDone) + initGlobalFeatures(); return new CWSCHelper(r, outputAllocator, authToken, wscMode, clientCert, logctx, roxieAbortMonitor, STsoap); } IWSCHelper * createHttpCallHelper(IWSCRowProvider *r, IEngineRowAllocator * outputAllocator, const char *authToken, WSCMode wscMode, ClientCertificate *clientCert, const IContextLogger &logctx, IRoxieAbortMonitor * roxieAbortMonitor) { + if (!globalFeaturesInitDone) + initGlobalFeatures(); return new CWSCHelper(r, outputAllocator, authToken, wscMode, clientCert, logctx, roxieAbortMonitor, SThttp); } @@ -2060,7 +2129,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo read += bytesRead; response.setLength(read); if (bytesRead==0) { - master->logctx.CTXLOG("%sCALL: Warning %sHTTP response terminated prematurely",master->wscType == STsoap ? "SOAP" : "HTTP",chunked?"CHUNKED ":""); + master->logctx.CTXLOG("%s: Warning %sHTTP response terminated prematurely", getWsCallTypeName(master->wscType),chunked?"CHUNKED ":""); break; // oops looks likesocket closed early } } @@ -2087,9 +2156,9 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (checkContentDecoding(dbgheader, response, contentEncoding)) decodeContent(contentEncoding.str(), response); if (soapTraceLevel > 6 || master->logXML) - master->logctx.mCTXLOG("%sCALL: LEN=%d %sresponse(%s%s)", master->wscType == STsoap ? "SOAP" : "HTTP",response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); + master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", dbgheader.str(), response.str()); else if (soapTraceLevel > 8) - master->logctx.mCTXLOG("%sCALL: LEN=%d %sresponse(%s)", master->wscType == STsoap ? "SOAP" : "HTTP",response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but... + master->logctx.mCTXLOG("%s: LEN=%d %sresponse(%s)", getWsCallTypeName(master->wscType),response.length(),chunked?"CHUNKED ":"", response.str()); // not sure this is that useful but... return rval; } @@ -2236,7 +2305,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo inline void checkTimeLimitExceeded(unsigned * remainingMS) { if (master->isTimeLimitExceeded(remainingMS)) - throw MakeStringException(TIMELIMIT_EXCEEDED, "%sCALL TIMELIMIT(%ums) exceeded", master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + throw MakeStringException(TIMELIMIT_EXCEEDED, "%s TIMELIMIT(%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); } inline bool checkKeepAlive(StringBuffer& headers) @@ -2357,7 +2426,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo { if (master->timeLimitExceeded) { - master->logctx.CTXLOG("%sCALL exiting: time limit (%ums) exceeded",master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + master->logctx.CTXLOG("%s exiting: time limit (%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); processException(url, inputRows, e); return; } @@ -2365,7 +2434,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (e->errorCode() == ROXIE_ABORT_EVENT) { StringBuffer s; - master->logctx.CTXLOG("%sCALL exiting: Roxie Abort : %s",master->wscType == STsoap ? "SOAP" : "HTTP",e->errorMessage(s).str()); + master->logctx.CTXLOG("%s exiting: Roxie Abort : %s", getWsCallTypeName(master->wscType),e->errorMessage(s).str()); throw; } @@ -2390,7 +2459,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo checkRoxieAbortMonitor(master->roxieAbortMonitor); socket->write(request.str(), request.length()); if (soapTraceLevel > 4) - master->logctx.CTXLOG("%sCALL: sent request (%s) to %s:%d", master->wscType == STsoap ? "SOAP" : "HTTP",master->service.str(), url.host.str(), url.port); + master->logctx.CTXLOG("%s: sent request (%s) to %s:%d", getWsCallTypeName(master->wscType),master->service.str(), url.host.str(), url.port); checkTimeLimitExceeded(&remainingMS); checkRoxieAbortMonitor(master->roxieAbortMonitor); @@ -2400,7 +2469,7 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo keepAlive = keepAlive && keepAlive2; if (soapTraceLevel > 4) - master->logctx.CTXLOG("%sCALL: received response (%s) from %s:%d", master->wscType == STsoap ? "SOAP" : "HTTP",master->service.str(), url.host.str(), url.port); + master->logctx.CTXLOG("%s: received response (%s) from %s:%d", getWsCallTypeName(master->wscType),master->service.str(), url.host.str(), url.port); if (rval != 200) { @@ -2469,14 +2538,14 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo if (master->timeLimitExceeded) { processException(url, inputRows, e); - master->logctx.CTXLOG("%sCALL exiting: time limit (%ums) exceeded", master->wscType == STsoap ? "SOAP" : "HTTP", master->timeLimitMS); + master->logctx.CTXLOG("%s exiting: time limit (%ums) exceeded", getWsCallTypeName(master->wscType), master->timeLimitMS); break; } if (e->errorCode() == ROXIE_ABORT_EVENT) { StringBuffer s; - master->logctx.CTXLOG("%sCALL exiting: Roxie Abort : %s",master->wscType == STsoap ? "SOAP" : "HTTP",e->errorMessage(s).str()); + master->logctx.CTXLOG("%s exiting: Roxie Abort : %s", getWsCallTypeName(master->wscType),e->errorMessage(s).str()); throw; } @@ -2519,7 +2588,5 @@ class CWSCAsyncFor : implements IWSCAsyncFor, public CInterface, public CAsyncFo IWSCAsyncFor * createWSCAsyncFor(CWSCHelper * _master, IXmlWriterExt &_xmlWriter, ConstPointerArray &_inputRows, PTreeReaderOptions _options) { - if (!persistentInitDone) - initPersistentHandler(); return new CWSCAsyncFor(_master, _xmlWriter, _inputRows, _options); } diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp index 467b9f3d99c..ad13fd22bda 100644 --- a/common/workunit/workunit.cpp +++ b/common/workunit/workunit.cpp @@ -13885,7 +13885,7 @@ extern WORKUNIT_API void associateLocalFile(IWUQuery * query, WUFileType type, c else { StringBuffer hostname; - queryHostIP().getIpText(hostname); + queryHostIP().getHostText(hostname); query->addAssociatedFile(type, fullPathName, hostname, description, crc, minActivity, maxActivity); } } @@ -14451,7 +14451,7 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP myep.serialize(msg); // only used for tracing if (!conversation->send(msg)) { StringBuffer s("Failed to send query to Thor on "); - thorMaster.getUrlStr(s); + thorMaster.getEndpointHostText(s); throw MakeStringExceptionDirect(-1, s.str()); // maybe retry? } unsigned __int64 blockedTime = elapsedTimer.elapsedNs(); @@ -14461,21 +14461,21 @@ void executeThorGraph(const char * graphName, IConstWorkUnit &workunit, const IP } StringBuffer eps; - PROGLOG("Thor on %s running %s", thorMaster.getUrlStr(eps).str(), jobName.str()); + PROGLOG("Thor on %s running %s", thorMaster.getEndpointHostText(eps).str(), jobName.str()); MemoryBuffer reply; try { if (!conversation->recv(reply,INFINITE)) { StringBuffer s("Failed to receive reply from thor "); - thorMaster.getUrlStr(s); + thorMaster.getEndpointHostText(s); throw MakeStringExceptionDirect(-1, s.str()); } } catch (IException *e) { StringBuffer s("Failed to receive reply from thor "); - thorMaster.getUrlStr(s); + thorMaster.getEndpointHostText(s); s.append("; (").append(e->errorCode()).append(", "); e->errorMessage(s).append(")"); e->Release(); diff --git a/common/workunit/wujobq.cpp b/common/workunit/wujobq.cpp index 0a08bd9e77a..0a5bd0576a1 100644 --- a/common/workunit/wujobq.cpp +++ b/common/workunit/wujobq.cpp @@ -109,7 +109,7 @@ class CJobQueueItem: implements IJobQueueItem, public CInterface item->setProp("@wuid",qi->queryWUID()); item->setProp("@owner",qi->queryOwner()); StringBuffer eps; - qi->queryEndpoint().getUrlStr(eps); + getRemoteAccessibleHostText(eps, qi->queryEndpoint()); item->setProp("@node",eps.str()); StringBuffer dts; qi->queryEnqueuedTime().getString(dts); @@ -1092,7 +1092,7 @@ class CJobQueue: public CJobQueueBase, implements IJobQueue ret = qd.root->addPropTree("Client"); ret->setPropInt64("@session",sessionid); StringBuffer eps; - ret->setProp("@node",queryMyNode()->endpoint().getUrlStr(eps).str()); + ret->setProp("@node",queryMyNode()->endpoint().getEndpointHostText(eps).str()); } return ret; } diff --git a/dali/base/daclient.cpp b/dali/base/daclient.cpp index 714e70bdcb6..b8ef382995b 100644 --- a/dali/base/daclient.cpp +++ b/dali/base/daclient.cpp @@ -231,7 +231,7 @@ CSDSServerStatus::CSDSServerStatus(const char *servername) IPropertyTree &root = *conn->queryRoot(); root.setProp("@name",servername); StringBuffer node; - queryMyNode()->endpoint().getIpText(node); + queryMyNode()->endpoint().getHostText(node); root.setProp("@node",node.str()); root.setPropInt("@mpport",queryMyNode()->endpoint().port); CDateTime dt; @@ -336,7 +336,7 @@ bool updateDaliEnv(IPropertyTree *env, bool forceGroupUpdate, const char *daliIp if (querySDS().updateEnvironment(env, forceGroupUpdate, response)) { StringBuffer tmp; - PROGLOG("Environment and node groups updated in dali at %s",daliep.getUrlStr(tmp).str()); + PROGLOG("Environment and node groups updated in dali at %s",daliep.getEndpointHostText(tmp).str()); } else ret = false; diff --git a/dali/base/dacoven.cpp b/dali/base/dacoven.cpp index 1ba5324da8c..30cd69fa001 100644 --- a/dali/base/dacoven.cpp +++ b/dali/base/dacoven.cpp @@ -163,7 +163,7 @@ static void checkDaliVersionInfo(ICommunicator *comm, CDaliVersion &serverVersio mb.append(ClientVersion); mb.append(MinServerVersion); StringBuffer daliEpStr; - comm->queryGroup().queryNode(0).endpoint().getUrlStr(daliEpStr); // NB: there's always exactly 1 node + comm->queryGroup().queryNode(0).endpoint().getEndpointHostText(daliEpStr); // NB: there's always exactly 1 node if (!comm->sendRecv(mb, RANK_RANDOM, MPTAG_DALI_COVEN_REQUEST, VERSION_REQUEST_TIMEOUT)) throw makeStringExceptionV(-1, "Failed retrieving version information from server [%s], legacy server?", daliEpStr.str()); if (!mb.length()) @@ -206,7 +206,7 @@ static CriticalSection foreignDaliVersionCacheCrit; void checkForeignDaliVersionInfo(const INode *foreignDali, CDaliVersion &serverVersion, CDaliVersion &minClientVersion) { StringBuffer foreignDaliStr; - foreignDali->endpoint().getUrlStr(foreignDaliStr); + foreignDali->endpoint().getEndpointHostText(foreignDaliStr); CriticalBlock b(foreignDaliVersionCacheCrit); std::pair result; @@ -395,7 +395,7 @@ CDaliUidAllocator &CDaliUidAllocator::find(CIArrayOf &uidallo uidallocators.append(*ret); } StringBuffer eps; - DBGLOG("Added foreign UID allocator for %s", ret->node.getUrlStr(eps).str()); + DBGLOG("Added foreign UID allocator for %s", ret->node.getEndpointHostText(eps).str()); return *ret; } @@ -498,7 +498,7 @@ class CCovenServer: public CCovenBase store->serialize(mb.clear()); if (!sendRecv(mb,r,MPTAG_DALI_COVEN_REQUEST, COVEN_SERVER_TIMEOUT)) { StringBuffer str; - throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getUrlStr(str).str()); + throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getEndpointHostText(str).str()); } mergeStore(store,mb,true); } @@ -508,7 +508,7 @@ class CCovenServer: public CCovenBase for (;;) { if (!recv(mb,r,MPTAG_DALI_COVEN_REQUEST,&sender,COVEN_SERVER_TIMEOUT)) { - throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getUrlStr(str).str()); + throw MakeStringException(-1,"Could not connect to %s",grp->queryNode(r).endpoint().getEndpointHostText(str).str()); } if (RANK_NULL==sender) processMessage(mb); diff --git a/dali/base/dadfs.cpp b/dali/base/dadfs.cpp index c2f5f587733..b10debff147 100644 --- a/dali/base/dadfs.cpp +++ b/dali/base/dadfs.cpp @@ -289,7 +289,7 @@ class DECL_EXCEPTION CDFS_Exception: implements IDFS_Exception, public CInterfac case DFSERR_LookupAccessDenied: { StringBuffer ip; - queryCoven().queryGroup().queryNode(0).endpoint().getIpText(ip); + queryCoven().queryGroup().queryNode(0).endpoint().getHostText(ip); return str.appendf(" Lookup access denied for scope %s at Dali %s", errstr.str(), ip.str()); } case DFSERR_CreateAccessDenied: @@ -811,7 +811,7 @@ static void foreignDaliSendRecv(const INode *foreigndali,CMessageBuffer &mb, uns Owned comm = createCommunicator(grp,true); if (!comm->verifyConnection(0,foreigndalitimeout)) { StringBuffer tmp; - IDFS_Exception *e = new CDFS_Exception(DFSERR_ForeignDaliTimeout, foreigndali->endpoint().getUrlStr(tmp).str()); + IDFS_Exception *e = new CDFS_Exception(DFSERR_ForeignDaliTimeout, foreigndali->endpoint().getEndpointHostText(tmp).str()); throw e; } comm->sendRecv(mb,0,MPTAG_DFS_REQUEST); @@ -3394,7 +3394,7 @@ class CDistributedFile: public CDistributedFileBase const char *grp = root->queryProp("@group"); if (!grp||!*grp) { StringBuffer eps; - pt->addProp("@node",part.queryNode(0)->endpoint().getUrlStr(eps).str()); // legacy + pt->addProp("@node",part.queryNode(0)->endpoint().getEndpointHostText(eps).str()); // legacy } const char *override = part.queryOverrideName(); if (override&&*override) @@ -4319,7 +4319,7 @@ protected: friend class CDistributedFilePart; CriticalBlock block(errcrit); StringBuffer s("Failed to find file part "); s.append(partfile->queryFilename()).append(" on "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getEndpointHostText(s); EXCLOG(e, s.str()); e->Release(); } @@ -7748,7 +7748,7 @@ class CNamedGroupStore: implements INamedGroupStore, public CInterface ForEachItemIn(e, epa) { StringBuffer ipStr; - epa.item(e).getIpText(ipStr); + epa.item(e).getHostText(ipStr); IPropertyTree *n = val->addPropTree("Node"); n->setProp("@ip", ipStr); } @@ -7774,7 +7774,7 @@ class CNamedGroupStore: implements INamedGroupStore, public CInterface std::vector ips; while (true) { - gi->query().endpoint().getIpText(ipStr.clear()); + gi->query().endpoint().getHostText(ipStr.clear()); ips.push_back(ipStr.str()); if (!gi->next()) break; @@ -8957,7 +8957,7 @@ void CDistributedFileDirectory::fixDates(IDistributedFile *file) CriticalBlock block(crit); StringBuffer s("Failed to find file part "); s.append(partfile->queryFilename()).append(" on "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getEndpointHostText(s); EXCLOG(e, s.str()); e->Release(); } @@ -11334,7 +11334,7 @@ class CDaliDFSServer: public Thread, public CTransactionLogTracker, implements I if (block0.slow()) { SocketEndpoint ep = mb.getSender(); - ep.getUrlStr(block0.appendMsg(trc).append(" from ")); + ep.getEndpointHostText(block0.appendMsg(trc).append(" from ")); } } @@ -11939,10 +11939,10 @@ bool removePhysicalFiles(IGroup *grp,const char *_filemask,unsigned short port,C PROGLOG("Removed '%s'",partfile->queryFilename()); unsigned t = msTick()-start; if (t>5*1000) - DBGLOG("Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000); + DBGLOG("Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } else - IWARNLOG("Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + IWARNLOG("Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); #else if (partfile->exists()) PROGLOG("Would remove '%s'",partfile->queryFilename()); @@ -11957,7 +11957,7 @@ bool removePhysicalFiles(IGroup *grp,const char *_filemask,unsigned short port,C else { StringBuffer s("Failed to remove file part "); s.append(partfile->queryFilename()).append(" from "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getEndpointHostText(s); EXCLOG(e, s.str()); e->Release(); } @@ -13651,12 +13651,12 @@ bool CDistributedFileDirectory::removePhysicalPartFiles(const char *logicalName, { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)&&!islazy) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000); + LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } catch (IException *e) @@ -13668,7 +13668,7 @@ bool CDistributedFileDirectory::removePhysicalPartFiles(const char *logicalName, { StringBuffer s("Failed to remove file part "); s.append(partfile->queryFilename()).append(" from "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getEndpointHostText(s); EXCLOG(e, s.str()); e->Release(); } diff --git a/dali/base/dafdesc.cpp b/dali/base/dafdesc.cpp index d624b0c3f62..42f77372327 100644 --- a/dali/base/dafdesc.cpp +++ b/dali/base/dafdesc.cpp @@ -869,7 +869,7 @@ protected: friend class CFileDescriptor; SocketEndpoint ep = queryNode(0)->endpoint(); StringBuffer tmp; if (!ep.isNull()) - pt->setProp("@node",ep.getUrlStr(tmp).str()); + pt->setProp("@node",ep.getEndpointHostText(tmp).str()); if (overridename.isEmpty()&&!parent.partmask.isEmpty()) { expandMask(tmp.clear(), parent.partmask, 0, 1); pt->setProp("@name",tmp.str()); @@ -3039,10 +3039,10 @@ void removePartFiles(IFileDescriptor *desc,IMultiException *mexcept) // PROGLOG("Removed '%s'",partfile->queryFilename()); unsigned t = msTick()-start; if (t>60*1000) - OWARNLOG("Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000); + OWARNLOG("Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } // else -// OWARNLOG("Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); +// OWARNLOG("Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); } catch (IException *e) { @@ -3051,7 +3051,7 @@ void removePartFiles(IFileDescriptor *desc,IMultiException *mexcept) else { StringBuffer s("Failed to remove file part "); s.append(partfile->queryFilename()).append(" from "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getEndpointHostText(s); EXCLOG(e, s.str()); e->Release(); } @@ -3323,7 +3323,7 @@ void extractFilePartInfo(IPropertyTree &info, IFileDescriptor &file) IPropertyTree *copyTree = partTree->addPropTree("Copy", createPTree()); copyTree->setProp("@filePath", rfn.getLocalPath(path.clear())); - copyTree->setProp("@host", rfn.queryEndpoint().getUrlStr(host.clear())); + copyTree->setProp("@host", rfn.queryEndpoint().getEndpointHostText(host.clear())); } } } diff --git a/dali/base/dasds.cpp b/dali/base/dasds.cpp index 87213719dad..6a3055903fb 100644 --- a/dali/base/dasds.cpp +++ b/dali/base/dasds.cpp @@ -3459,7 +3459,7 @@ class CLock : implements IInterface, public CInterface else { StringBuffer nodeStr; - node->endpoint().getUrlStr(nodeStr); + node->endpoint().getEndpointHostText(nodeStr); PROGLOG("Validating connection to %s", nodeStr.str()); if (!queryWorldCommunicator().verifyConnection(node, LOCKSESSCHECK)) { @@ -4056,7 +4056,7 @@ int CSDSTransactionServer::run() StringBuffer s; mb.append(e->errorMessage(s).str()); StringBuffer clientUrl("EXCEPTION in reply to client "); - mb.getSender().getUrlStr(clientUrl); + mb.getSender().getEndpointHostText(clientUrl); EXCLOG(e, clientUrl.str(), MSGCLS_warning); e->Release(); } @@ -4081,7 +4081,7 @@ int CSDSTransactionServer::run() catch (IException *e) { StringBuffer s("Failure receiving message from client "); - mb.getSender().getUrlStr(s); + mb.getSender().getEndpointHostText(s); IWARNLOG(e, s.str()); e->Release(); } @@ -4784,7 +4784,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) mb.append(e->errorCode()); mb.append(e->errorMessage(s.clear())); StringBuffer clientUrl("EXCEPTION in reply to client "); - mb.getSender().getUrlStr(clientUrl); + mb.getSender().getEndpointHostText(clientUrl); EXCLOG(e, clientUrl.str(), MSGCLS_warning); e->Release(); } @@ -4797,7 +4797,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) mb.append(e->errorCode()); mb.append(e->errorMessage(s).str()); StringBuffer clientUrl("EXCEPTION in reply to client "); - mb.getSender().getUrlStr(clientUrl); + mb.getSender().getEndpointHostText(clientUrl); LOG(MCoperatorError, unknownJob, e); } try { @@ -4818,7 +4818,7 @@ void CSDSTransactionServer::processMessage(CMessageBuffer &mb) StringBuffer s; mb.append(e->errorMessage(s).str()); StringBuffer clientUrl("EXCEPTION in reply to client "); - mb.getSender().getUrlStr(clientUrl); + mb.getSender().getEndpointHostText(clientUrl); EXCLOG(e, clientUrl.str(), MSGCLS_warning); e->Release(); try diff --git a/dali/base/dasess.cpp b/dali/base/dasess.cpp index 471e2b0ec9e..ba4a1e58b57 100644 --- a/dali/base/dasess.cpp +++ b/dali/base/dasess.cpp @@ -320,7 +320,7 @@ class CProcessSessionState: public CSessionState StringBuffer &getDetails(StringBuffer &buf) { StringBuffer ep; - return buf.appendf("%16" I64F "X: %s, role=%s",CSessionState::id,node->endpoint().getUrlStr(ep).str(),queryRoleName(role)); + return buf.appendf("%16" I64F "X: %s, role=%s",CSessionState::id,node->endpoint().getEndpointHostText(ep).str(),queryRoleName(role)); } void addSessionIds(CProcessSessionState &other, bool prevOnly) { @@ -1101,7 +1101,7 @@ class CClientSessionManager: public CSessionManagerBase, implements IConnectionM Owned node = createINode(ep); if (queryCoven().inCoven(node)) { StringBuffer str; - PROGLOG("Coven Session Stopping (%s)",ep.getUrlStr(str).str()); + PROGLOG("Coven Session Stopping (%s)",ep.getEndpointHostText(str).str()); if (queryCoven().size()==1) notifyServerStopped(true); } @@ -1325,7 +1325,7 @@ class CCovenSessionManager: public CSessionManagerBase, implements ISessionManag void addProcessSession(SessionId id,INode *client,DaliClientRole role) { StringBuffer str; - PROGLOG("Session starting %" I64F "x (%s) : role=%s",id,client->endpoint().getUrlStr(str).str(),queryRoleName(role)); + PROGLOG("Session starting %" I64F "x (%s) : role=%s",id,client->endpoint().getEndpointHostText(str).str(),queryRoleName(role)); CHECKEDCRITICALBLOCK(sessmanagersect,60000); CProcessSessionState *s = new CProcessSessionState(id,client,role); while (!sessionstates.add(s)) // takes ownership @@ -1737,7 +1737,7 @@ class CCovenSessionManager: public CSessionManagerBase, implements ISessionManag void onClose(SocketEndpoint &ep) { StringBuffer clientStr; - PROGLOG("Client closed (%s)", ep.getUrlStr(clientStr).str()); + PROGLOG("Client closed (%s)", ep.getEndpointHostText(clientStr).str()); SessionId idtostop; { @@ -1787,7 +1787,7 @@ class CCovenSessionManager: public CSessionManagerBase, implements ISessionManag if (state) { const CProcessSessionState *pstate = QUERYINTERFACE(state,const CProcessSessionState); if (pstate) - return pstate->queryNode().endpoint().getUrlStr(buf); + return pstate->queryNode().endpoint().getEndpointHostText(buf); } return buf; } @@ -1957,14 +1957,14 @@ bool registerClientProcess(ICommunicator *comm, IGroup *& retcoven,unsigned time if (lastNextLog) // see below, true if has been round timeout loop. Only output confirmation, if have issued 'Failed to connect ..' messages { StringBuffer str("Connected to Dali Server "); - comm->queryGroup().queryNode(r).endpoint().getUrlStr(str); + comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str); LOG(MCoperatorProgress, "%s", str.str()); } return true; } } StringBuffer str; - OERRLOG("Waiting for Dali to be available - server: %s", comm->queryGroup().queryNode(r).endpoint().getUrlStr(str).str()); + OERRLOG("Waiting for Dali to be available - server: %s", comm->queryGroup().queryNode(r).endpoint().getEndpointHostText(str).str()); if (tm.timedout()) { PROGLOG("%s", str.append(" Timed out.").str()); diff --git a/dali/base/dasubs.cpp b/dali/base/dasubs.cpp index da474417cde..4b040530c92 100644 --- a/dali/base/dasubs.cpp +++ b/dali/base/dasubs.cpp @@ -91,7 +91,7 @@ class CSubscriptionStub: implements ISubscription, public CInterface // Kludge to avoid locking SDS on blocked client hasaborted = true; StringBuffer tmp; - throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getUrlStr(tmp).str()); + throw MakeStringException(-1,"Subscription notification to %s timed out",dst->endpoint().getEndpointHostText(tmp).str()); return; } @@ -123,7 +123,7 @@ class CSubscriptionStub: implements ISubscription, public CInterface StringBuffer &getDetails(StringBuffer &buf) { StringBuffer ep; - return buf.appendf("%16" I64F "X: %s %s",sid,dst->endpoint().getUrlStr(ep).str(),hasaborted?"aborted":""); + return buf.appendf("%16" I64F "X: %s %s",sid,dst->endpoint().getEndpointHostText(ep).str(),hasaborted?"aborted":""); } }; @@ -275,7 +275,7 @@ class CDaliPublisherServer: public IDaliServer, public Thread, public CDaliPubli serializeException(exception, mb); while (!coven.reply(mb,60000)) { StringBuffer eps; - DBGLOG("MSR_ADD_SUBSCRIPTION_PRIMARY reply timed out to %s try %d",mb.getSender().getUrlStr(eps).str(),retry+1); + DBGLOG("MSR_ADD_SUBSCRIPTION_PRIMARY reply timed out to %s try %d",mb.getSender().getEndpointHostText(eps).str(),retry+1); if (retry++==3) return; } diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp index f14a8df714a..fc9882c67a3 100644 --- a/dali/base/dautils.cpp +++ b/dali/base/dautils.cpp @@ -300,12 +300,12 @@ class CMultiDLFN if (sub.isForeign()) { tmp.append(FOREIGN_SCOPE "::"); - foreignEp.getUrlStr(tmp).append("::"); + foreignEp.getEndpointHostText(tmp).append("::"); } else if (sub.isRemote()) { tmp.append(REMOTE_SCOPE "::"); - foreignEp.getUrlStr(tmp).append("::"); + foreignEp.getEndpointHostText(tmp).append("::"); } tmp.append(name); lfnExpanded.append(tmp.str()); @@ -744,7 +744,7 @@ void CDfsLogicalFileName::normalizeName(const char *name, StringAttr &res, bool normalizeNodeName(s1, ns1-s1, foreignep, strict); if (!foreignep.isNull()) { - foreignep.getUrlStr(str.append("::")); + foreignep.getEndpointHostText(str.append("::")); s = ns1; localpos = str.length()+2; } @@ -830,7 +830,7 @@ bool CDfsLogicalFileName::normalizeExternal(const char * name, StringAttr &res, if (ep.isNull()) return false; - ep.getUrlStr(str.append("::")); + ep.getEndpointHostText(str.append("::")); if (ns1[2] == '>') { str.append("::"); @@ -984,7 +984,7 @@ void CDfsLogicalFileName::setForeign(const SocketEndpoint &daliep,bool checkloca if (isExternal()||(checklocal&&isForeign())) return; StringBuffer str(FOREIGN_SCOPE "::"); - daliep.getUrlStr(str); + daliep.getEndpointHostText(str); str.append("::"); str.append(get(true)); set(str); @@ -1167,7 +1167,7 @@ void CDfsLogicalFileName::setExternal(const char *location,const char *path) void CDfsLogicalFileName::setExternal(const SocketEndpoint &dafsip,const char *path) { StringBuffer str; - dafsip.getUrlStr(str); + dafsip.getEndpointHostText(str); setExternal(str.str(),path); } @@ -1198,7 +1198,7 @@ void CDfsLogicalFileName::setQuery(const char *location,const char *query) void CDfsLogicalFileName::setQuery(const SocketEndpoint &rfsep,const char *query) { StringBuffer str; - rfsep.getUrlStr(str); + rfsep.getEndpointHostText(str); setQuery(str.str(),query); } @@ -1992,7 +1992,7 @@ void expandFileTree(IPropertyTree *file,bool expandnodes,const char *cluster) done.replace(true,num); INode *node = clusterinfo?clusterinfo->queryNode(num,max,0):&grp->queryNode(num%grp->ordinality()); if (node) { - node->endpoint().getIpText(ips.clear()); + node->endpoint().getHostText(ips.clear()); iter->query().setProp("@node",ips.str()); } } @@ -3423,7 +3423,7 @@ class CLockInfo : public CSimpleInterfaceOf { CLockMetaData &lD = *ldInfo.item(c); SocketEndpoint ep(lD.queryEp()); - ep.getIpText(ipStr.clear()); + ep.getHostText(ipStr.clear()); if (!WildMatch(ipStr, ipPattern)) ldInfo.zap(&lD); } diff --git a/dali/base/dautils.hpp b/dali/base/dautils.hpp index 8f9aae1ba75..10c145de248 100644 --- a/dali/base/dautils.hpp +++ b/dali/base/dautils.hpp @@ -210,7 +210,7 @@ struct da_decl TransactionLog owner.startTransaction(cmd); owner.getCmdText(cmd, msg); msg.append(", endpoint="); - ep.getUrlStr(msg); + ep.getEndpointHostText(msg); startCycles = get_cycles_now(); } else diff --git a/dali/daliadmin/daadmin.cpp b/dali/daliadmin/daadmin.cpp index 91f11f4a1d6..26954d26776 100644 --- a/dali/daliadmin/daadmin.cpp +++ b/dali/daliadmin/daadmin.cpp @@ -628,7 +628,7 @@ static void writeGroup(IGroup *group, const char *name, const char *outputFilena StringBuffer eps; for (unsigned i=0;iordinality();i++) { - group->queryNode(i).endpoint().getUrlStr(eps.clear()); + group->queryNode(i).endpoint().getEndpointHostText(eps.clear()); if (io) { eps.newline(); @@ -1014,7 +1014,7 @@ int dfsverify(const char *name,CDateTime *cutoff, IUserDescriptor *user) SocketEndpoint ep(part->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); UERRLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); return 4; } @@ -1059,13 +1059,13 @@ int dfsverify(const char *name,CDateTime *cutoff, IUserDescriptor *user) try { partfile.setown(createIFile(rfn)); - // OUTLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + // OUTLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); if (partfile) { CriticalUnblock unblock(crit); item.crc = partfile->getCRC(); partfile->getTime(NULL,&item.dt,NULL); if ((item.crc==0)&&!partfile->exists()) { - UERRLOG("VERIFY: does not exist part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + UERRLOG("VERIFY: does not exist part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); ok = false; } } @@ -1076,7 +1076,7 @@ int dfsverify(const char *name,CDateTime *cutoff, IUserDescriptor *user) catch (IException *e) { StringBuffer s; - s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); EXCLOG(e, s.str()); e->Release(); ok = false; @@ -2192,7 +2192,7 @@ void daliping(const char *dalis,unsigned connecttime,unsigned n) StringBuffer qname("TESTINGQ_"); SocketEndpoint ep; ep.setLocalHost(0); - ep.getUrlStr(qname); + ep.getEndpointHostText(qname); Owned qconn; qconn.setown(createNamedQueueConnection(0)); Owned channel; @@ -3058,9 +3058,9 @@ void migrateFiles(const char *srcGroup, const char *tgtGroup, const char *filema Owned iFileIOStream = getFileIOStream(relPos+1); StringBuffer outputLine; - srcEp.getIpText(outputLine); + srcEp.getHostText(outputLine); outputLine.append(","); - tgtEp.getIpText(outputLine); + tgtEp.getHostText(outputLine); outputLine.append(","); IPartDescriptor *part = fileDesc->queryPart(partNum); diff --git a/dali/daliadmin/daliadmin.cpp b/dali/daliadmin/daliadmin.cpp index d70ceb728ed..474580c01d4 100644 --- a/dali/daliadmin/daliadmin.cpp +++ b/dali/daliadmin/daliadmin.cpp @@ -159,7 +159,7 @@ int main(int argc, const char* argv[]) (memcmp(param,"timeout=",8)==0)) props->loadProp(param); else if ((i==1)&&(isdigit(*param)||(*param=='.'))&&ep.set(((*param=='.')&¶m[1])?(param+1):param,DALI_SERVER_PORT)) - props->setProp("server",ep.getUrlStr(tmps.clear()).str()); + props->setProp("server",ep.getEndpointHostText(tmps.clear()).str()); else { if ((strieq(param,"help")) || (strieq(param,"-help")) || (strieq(param,"--help"))) { usage(argv[0]); diff --git a/dali/dalidiag/dalidiag.cpp b/dali/dalidiag/dalidiag.cpp index 9fc16b59607..33b74be6af7 100644 --- a/dali/dalidiag/dalidiag.cpp +++ b/dali/dalidiag/dalidiag.cpp @@ -119,7 +119,7 @@ void timeQorSDS(bool timeq) if (timeq) { StringBuffer qname("TESTINGQ_"); - ep.getUrlStr(qname); + ep.getEndpointHostText(qname); qconn.setown(createNamedQueueConnection(0)); channel.setown(qconn->open(qname.str())); while (channel->probe()) { @@ -132,7 +132,7 @@ void timeQorSDS(bool timeq) } else { subname.append("TESTSDS_"); - ep.getUrlStr(subname); + ep.getEndpointHostText(subname); StringBuffer fullname("TESTING/"); fullname.append(subname); sdssubid = querySDS().subscribe(fullname.str(), testsdssub); @@ -379,9 +379,9 @@ void backupList(const char *cluster) rank_t r; StringBuffer str; for (r=0;rqueryNode(r).endpoint().getUrlStr(str.clear()); + group->queryNode(r).endpoint().getEndpointHostText(str.clear()); str.append(' '); - group->queryNode((r+1)%n).endpoint().getUrlStr(str); + group->queryNode((r+1)%n).endpoint().getEndpointHostText(str); printf("%s\n",str.str()); } } diff --git a/dali/datest/datest.cpp b/dali/datest/datest.cpp index 40a739caca3..c0c7ecaf50d 100644 --- a/dali/datest/datest.cpp +++ b/dali/datest/datest.cpp @@ -382,7 +382,7 @@ void testCDfsLogicalFileName() lfn=cdlfn.get(); assertex(strcmp(lfn,"file::10.150.10.75::c$::test::file.xyz")==0); lfn=cdlfn.get(true); assertex(strcmp(lfn,"file::10.150.10.75::c$::test::file.xyz")==0); verifyex(cdlfn.getEp(ep)); - ep.getUrlStr(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75")==0); + ep.getEndpointHostText(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75")==0); verifyex(cdlfn.getExternalPath(path.clear(),path,true)); assertex(strcmp(path.str(),"c:\\test\\file.xyz")==0); verifyex(cdlfn.getExternalPath(dir.clear(),path.clear(),true)); assertex(strcmp(path.str(),"file.xyz")==0); verifyex(cdlfn.getExternalPath(path.clear(),path,false)); assertex(strcmp(path.str(),"/c$/test/file.xyz")==0); @@ -393,7 +393,7 @@ void testCDfsLogicalFileName() assertex(cdlfn.isExternal()); lfn=cdlfn.get(); assertex(strcmp(lfn,"file::10.150.10.75:7100::c$::test::file.xyz")==0); verifyex(cdlfn.getEp(ep)); - ep.getUrlStr(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75:7100")==0); + ep.getEndpointHostText(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75:7100")==0); verifyex(cdlfn.getExternalPath(path.clear(),path,true)); assertex(strcmp(path.str(),"c:\\test\\file.xyz")==0); verifyex(cdlfn.getExternalPath(dir.clear(),path.clear(),true)); assertex(strcmp(path.str(),"file.xyz")==0); verifyex(cdlfn.getExternalPath(path.clear(),path,false)); assertex(strcmp(path.str(),"/c$/test/file.xyz")==0); @@ -403,7 +403,7 @@ void testCDfsLogicalFileName() assertex(cdlfn.isForeign()); assertex(!cdlfn.isExternal()); verifyex(cdlfn.getEp(ep)); - ep.getUrlStr(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75")==0); + ep.getEndpointHostText(eps.clear()); assertex(strcmp(eps.str(),"10.150.10.75")==0); lfn=cdlfn.get(); assertex(strcmp(lfn,"foreign::10.150.10.75::test::file.xyz")==0); lfn=cdlfn.get(true); assertex(strcmp(lfn,"test::file.xyz")==0); cdlfn.getScopes(scopes.clear()); assertex(strcmp(scopes.str(),"foreign::10.150.10.75::test")==0); @@ -1021,7 +1021,7 @@ void QTest(bool testput) #if 1 StringBuffer eps; if (i%100==99) { - PROGLOG("Got %s - %d from %s",str.get(),n,node->endpoint().getUrlStr(eps).str()); + PROGLOG("Got %s - %d from %s",str.get(),n,node->endpoint().getEndpointHostText(eps).str()); PROGLOG("time taken = %d",msTick()-t1); t1 = msTick(); } @@ -3292,13 +3292,13 @@ void testMultiConnect() void connected(unsigned idx,const SocketEndpoint &ep,ISocket *socket) { StringBuffer epstr; - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); printf("%s suceeded\n",epstr.str()); } void failed(unsigned idx,const SocketEndpoint &ep,int err) { StringBuffer epstr; - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); printf("%s failed (%d)\n",epstr.str(),err); } } notify; diff --git a/dali/datest/floodtest.cpp b/dali/datest/floodtest.cpp index 6c89a5d5fa4..1562fe864d7 100644 --- a/dali/datest/floodtest.cpp +++ b/dali/datest/floodtest.cpp @@ -845,7 +845,7 @@ void floodtestServer(const char *exename,const char *daliserver,const char *grpn StringAttr msg; mb.read(msg); StringBuffer url; - IERRLOG("%3d (%s): %s\n",r+1,group->queryNode(r).endpoint().getUrlStr(url).str(),msg.get()); + IERRLOG("%3d (%s): %s\n",r+1,group->queryNode(r).endpoint().getEndpointHostText(url).str(),msg.get()); } else if (fn==FLOODTEST_CONNECTED) { if (setState(group,"Connected",r,connected)) { diff --git a/dali/daunittest/dautdfs.cpp b/dali/daunittest/dautdfs.cpp index cfecb5f218f..87cd2c3fc63 100644 --- a/dali/daunittest/dautdfs.cpp +++ b/dali/daunittest/dautdfs.cpp @@ -279,9 +279,9 @@ class TTestDFS : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT(stricmp(s.str(),t.str())==0); StringBuffer eps; if (isrep) - rep.getUrlStr(eps); + rep.getEndpointHostText(eps); else - ep.getUrlStr(eps); + ep.getEndpointHostText(eps); t.insert(0,eps.str()); t.insert(0,"//"); t.appendf("/testfile3._%d_of_8",pi+1); @@ -340,9 +340,9 @@ class TTestDFS : public CPPUNIT_NS::TestFixture CPPUNIT_ASSERT(stricmp(s.str(),t.str())==0); StringBuffer eps; if (isrep) - rep.getUrlStr(eps); + rep.getEndpointHostText(eps); else - ep.getUrlStr(eps); + ep.getEndpointHostText(eps); t.insert(0,eps.str()); t.insert(0,"//"); t.appendf("/testfile4._%d_of_8",pi+1); diff --git a/dali/dfu/dfurepl.cpp b/dali/dfu/dfurepl.cpp index 56102de5e37..f7b9c90a13c 100644 --- a/dali/dfu/dfurepl.cpp +++ b/dali/dfu/dfurepl.cpp @@ -301,7 +301,7 @@ struct ReplicateFileItem: extends CInterface IPropertyTree &root = *conn->queryRoot(); root.setProp("@name",lfn); StringBuffer node; - queryMyNode()->endpoint().getIpText(node); + queryMyNode()->endpoint().getHostText(node); root.setProp("@node",node.str()); root.setPropInt("@mpport",queryMyNode()->endpoint().port); dt.setNow(); diff --git a/dali/dfu/dfurun.cpp b/dali/dfu/dfurun.cpp index 991148ccd47..46828fdb4c8 100644 --- a/dali/dfu/dfurun.cpp +++ b/dali/dfu/dfurun.cpp @@ -91,7 +91,7 @@ class CDFUengine: public CInterface, implements IDFUengine ep.setLocalHost(0); StringBuffer aln; aln.append(",FileAccess,DfuPlus,").append(func).append(','); - ep.getUrlStr(aln); + ep.getEndpointHostText(aln); aln.append(','); if (userdesc) userdesc->getUserName(aln); @@ -466,7 +466,7 @@ class CDFUengine: public CInterface, implements IDFUengine throwError3(DFTERR_InvalidFilePath, pfilePath, dotDotString, dotString); StringBuffer netaddress; - filename.queryIP().getIpText(netaddress); + filename.queryIP().getHostText(netaddress); #ifdef _CONTAINERIZED Owned planes = getDropZonePlanesIterator(); ForEach(*planes) @@ -1020,7 +1020,7 @@ class CDFUengine: public CInterface, implements IDFUengine if (wuid.isEmpty()) return false; StringBuffer eps; - PROGLOG("%s: Copy %s from %s to %s",wuid.get(),srclfn,srcdali?srcdali->endpoint().getUrlStr(eps).str():"(local)",dstlfn); + PROGLOG("%s: Copy %s from %s to %s",wuid.get(),srclfn,srcdali?srcdali->endpoint().getEndpointHostText(eps).str():"(local)",dstlfn); DFUstate state = runWU(wuid); StringBuffer tmp; PROGLOG("%s: Done: %s",wuid.get(),encodeDFUstate(state,tmp).str()); @@ -1049,7 +1049,7 @@ class CDFUengine: public CInterface, implements IDFUengine Owned ftree = queryDistributedFileDirectory().getFileTree(srclfn,ctx.srcuser,srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign); if (!ftree.get()) { StringBuffer s; - throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } // now we can create name StringBuffer newroxieprefix; @@ -1124,7 +1124,7 @@ class CDFUengine: public CInterface, implements IDFUengine } else { StringBuffer s; - throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",srclfn,srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",srclfn,srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } if ((ctx.level==1)&&ctx.feedback) ctx.feedback->displaySummary("0",0); diff --git a/dali/dfu/dfurunkdp.cpp b/dali/dfu/dfurunkdp.cpp index 9574310c6ef..73a7699e0c0 100644 --- a/dali/dfu/dfurunkdp.cpp +++ b/dali/dfu/dfurunkdp.cpp @@ -214,7 +214,7 @@ static void runKDPNodes(const char *title,CIArrayOf &nodes) if (msTick()-start>15*60*1000) { StringBuffer err; err.append(title).append(" failed to start on node "); - it.queryEP().getUrlStr(err); + it.queryEP().getEndpointHostText(err); throw MakeStringException(-1, "%s", err.str()); } Sleep(5000); // no point in rushing when some left @@ -235,7 +235,7 @@ static void runKDPNodes(const char *title,CIArrayOf &nodes) if (msTick()-start>6*60*60*1000) { StringBuffer err; err.append(title).append(" failed to finish on node "); - it.queryEP().getUrlStr(err); + it.queryEP().getEndpointHostText(err); throw MakeStringException(-1, "%s", err.str()); } Sleep(5000); // no point in rushing when some left diff --git a/dali/dfu/dfuutil.cpp b/dali/dfu/dfuutil.cpp index 8122956f72c..85bfc3964c1 100644 --- a/dali/dfu/dfuutil.cpp +++ b/dali/dfu/dfuutil.cpp @@ -414,7 +414,7 @@ class CFileCloner else { StringBuffer s; - attrs.setProp("@cloneFrom", srcdali->endpoint().getUrlStr(s).str()); + attrs.setProp("@cloneFrom", srcdali->endpoint().getEndpointHostText(s).str()); attrs.setProp("@cloneFromDir", srcfdesc->queryDefaultDir()); if (srcCluster && *srcCluster) //where to copy from has been explicity set to a remote location, don't copy from local sources attrs.setProp("@cloneFromPeerCluster", "-"); @@ -642,7 +642,7 @@ class CFileCloner Owned ftree = fdir->getFileTree(slfn.get(),foreignuserdesc,srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign); if (!ftree.get()) { StringBuffer s; - throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } const char *dstlfn = slfn.get(); @@ -713,7 +713,7 @@ class CFileCloner } else { StringBuffer s; - throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",filename,srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",filename,srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } level--; } @@ -734,18 +734,18 @@ class CFileCloner Owned ftree = fdir->getFileTree(slfn.get(), foreignuserdesc, srcdali, FOREIGN_DALI_TIMEOUT, GetFileTreeOpts::appendForeign); if (!ftree.get()) { StringBuffer s; - throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } IPropertyTree *attsrc = ftree->queryPropTree("Attr"); if (!attsrc) { StringBuffer s; - throw MakeStringException(-1,"Attributes for source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Attributes for source file %s could not be found in Dali %s",slfn.get(),srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } CDfsLogicalFileName dlfn; dlfn.set(destfilename); if (strcmp(ftree->queryName(),queryDfsXmlBranchName(DXB_File))!=0) { StringBuffer s; - throw MakeStringException(-1,"Source file %s in Dali %s is not a simple file",filename,srcdali?srcdali->endpoint().getUrlStr(s).str():"(local)"); + throw MakeStringException(-1,"Source file %s in Dali %s is not a simple file",filename,srcdali?srcdali->endpoint().getEndpointHostText(s).str():"(local)"); } if (!srcdali.get()||queryCoven().inCoven(srcdali)) { // if dali is local and filenames same @@ -787,7 +787,7 @@ class CFileCloner { if (!daliNode) return "(local)"; - return daliNode->endpoint().getUrlStr(s).str(); + return daliNode->endpoint().getEndpointHostText(s).str(); } inline bool checkHasCluster(IDistributedFile *dfile) { @@ -833,7 +833,7 @@ class CFileCloner else { StringBuffer s; - if (checkValueChanged(dfile->queryAttributes().queryProp("@cloneFrom"), srcdali->endpoint().getUrlStr(s).str())) + if (checkValueChanged(dfile->queryAttributes().queryProp("@cloneFrom"), srcdali->endpoint().getEndpointHostText(s).str())) return true; if (checkValueChanged(dfile->queryAttributes().queryProp("@cloneFromDir"), srcfdesc->queryDefaultDir())) return true; @@ -1046,7 +1046,7 @@ class CDFUhelper: implements IDFUhelper, public CInterface out.append("\n"); ForEachNodeInGroup(i, *grp) { StringBuffer ip; - grp->getNode(i)->endpoint().getIpText(ip); + grp->getNode(i)->endpoint().getHostText(ip); out.append(" \n"); } out.append("\n"); @@ -1133,7 +1133,7 @@ class CDFUhelper: implements IDFUhelper, public CInterface Owned fdesc = queryDistributedFileDirectory().getFileDescriptor(srclfn, AccessMode::tbdRead, srcuser, node); if (!fdesc) { StringBuffer s; - throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getUrlStr(s).str()); + throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getEndpointHostText(s).str()); } Owned file = queryDistributedFileDirectory().createNew(fdesc); if (file) @@ -1169,7 +1169,7 @@ class CDFUhelper: implements IDFUhelper, public CInterface if (!ftree.get()) { StringBuffer s; - throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getUrlStr(s).str()); + throw MakeStringException(-1,"Source file %s could not be found in Dali %s",srclfn,daliep.getEndpointHostText(s).str()); } // first see if target exists (and remove if does and overwrite specified) Owned dfile = queryDistributedFileDirectory().lookup(lfn,user,AccessMode::tbdWrite,false,false,nullptr,defaultPrivilegedUser); @@ -1222,7 +1222,7 @@ class CDFUhelper: implements IDFUhelper, public CInterface else { StringBuffer s; - throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",srclfn,daliep.getUrlStr(s).str()); + throw MakeStringException(-1,"Source file %s in Dali %s is not a file or superfile",srclfn,daliep.getEndpointHostText(s).str()); } } diff --git a/dali/dfu/dfuwu.cpp b/dali/dfu/dfuwu.cpp index fcd3403e6a6..205361fe4d8 100644 --- a/dali/dfu/dfuwu.cpp +++ b/dali/dfu/dfuwu.cpp @@ -885,7 +885,7 @@ class CDFUmonitor: public CLinkedDFUWUchild, implements IDFUmonitor queryRoot()->removeProp("@handler"); else { StringBuffer s; - queryRoot()->setProp("@handler",ep.getUrlStr(s).str()); + queryRoot()->setProp("@handler",ep.getEndpointHostText(s).str()); } } @@ -932,9 +932,9 @@ static void printDesc(IFileDescriptor *desc) // PROGLOG(" groupWidth = %d",desc->queryClusterGroup(0)->ordinality()); // PROGLOG(" numSubFiles = %d",desc->getNumSubFiles()); // Owned group = desc->getGroup(0); -// PROGLOG(" group(0) = %d,%s,...,%s",group->ordinality(),group->queryNode(0).endpoint().getUrlStr(tmp1.clear()).str(),group->queryNode(group->ordinality()-1).endpoint().getUrlStr(tmp2.clear()).str()); +// PROGLOG(" group(0) = %d,%s,...,%s",group->ordinality(),group->queryNode(0).endpoint().getEndpointHostText(tmp1.clear()).str(),group->queryNode(group->ordinality()-1).endpoint().getEndpointHostText(tmp2.clear()).str()); // group.setown(desc->getGroup(1)); -// PROGLOG(" group(1) = %d,%s,...,%s",group->ordinality(),group->queryNode(0).endpoint().getUrlStr(tmp1.clear()).str(),group->queryNode(group->ordinality()-1).endpoint().getUrlStr(tmp2.clear()).str()); +// PROGLOG(" group(1) = %d,%s,...,%s",group->ordinality(),group->queryNode(0).endpoint().getEndpointHostText(tmp1.clear()).str(),group->queryNode(group->ordinality()-1).endpoint().getEndpointHostText(tmp2.clear()).str()); unsigned copy; for (copy = 0;copy<2;copy++) { unsigned i; @@ -1491,7 +1491,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec IPropertyTree * part = queryRoot()->setPropTree("Part",createPTree("Part")); part->setPropInt("@num",1); StringBuffer url; - rmfn.queryEndpoint().getUrlStr(url); + rmfn.queryEndpoint().getEndpointHostText(url); part->setProp("@node",url.str()); } @@ -1590,7 +1590,7 @@ class CDFUfileSpec: public CLinkedDFUWUchild, implements IDFUfileSpec // only used for source of copy IPropertyTree *t = queryUpdateProperties(); StringBuffer s; - t->setProp("@foreignDali",ep.getUrlStr(s).str()); + t->setProp("@foreignDali",ep.getEndpointHostText(s).str()); } bool getForeignDali(SocketEndpoint &ep) const @@ -3523,7 +3523,7 @@ IDfuFileCopier *createRemoteFileCopier(const char *qname,const char *clustername // should be no need for overwrite const char *wuid = wu->queryId(); StringBuffer eps; - PROGLOG("%s: Copy %s from %s to %s",wuid,srclfn,srcdali.getUrlStr(eps).str(),lfn); + PROGLOG("%s: Copy %s from %s to %s",wuid,srclfn,srcdali.getEndpointHostText(eps).str(),lfn); wuids.append(wuid); submitDFUWorkUnit(wu.getClear()); return true; diff --git a/dali/dfuXRefLib/XRefFilesNode.cpp b/dali/dfuXRefLib/XRefFilesNode.cpp index b5881acb93e..6eee332ceca 100644 --- a/dali/dfuXRefLib/XRefFilesNode.cpp +++ b/dali/dfuXRefLib/XRefFilesNode.cpp @@ -130,8 +130,8 @@ static bool checkPartsInCluster(const char *title,const char *clustername, IProp if (group->rank(ep)==RANK_NULL) { StringBuffer eps; - OERRLOG("%s %s Part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getUrlStr(eps).str(),clustername); - errstr.appendf("ERROR: %s %s part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getUrlStr(eps).str(),clustername); + OERRLOG("%s %s Part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getEndpointHostText(eps).str(),clustername); + errstr.appendf("ERROR: %s %s part %d on %s is not in cluster %s",title,rep?"Replicate":"Primary",pn,ep.getEndpointHostText(eps).str(),clustername); return false; } if (exists) diff --git a/dali/dfuXRefLib/dfurdir.cpp b/dali/dfuXRefLib/dfurdir.cpp index f6228bc1d31..9521be61edf 100644 --- a/dali/dfuXRefLib/dfurdir.cpp +++ b/dali/dfuXRefLib/dfurdir.cpp @@ -182,7 +182,7 @@ IPropertyTree *getDirectory(const char * directory, INode * node, unsigned short try { Owned dirTree = createPTree("machine"); StringBuffer url; - node->endpoint().getIpText(url); + node->endpoint().getHostText(url); dirTree->setProp("@ip", url.str()); XREFDirectoryBuilder builder; @@ -215,7 +215,7 @@ IPropertyTree *getDirectory(const char * directory, INode * node, unsigned short s.append(directory); if (node) { s.append(" on "); - node->endpoint().getUrlStr(s); + node->endpoint().getEndpointHostText(s); } if (port) s.append(" port ").append(port); diff --git a/dali/dfuXRefLib/dfuxreflib.cpp b/dali/dfuXRefLib/dfuxreflib.cpp index 30880557d91..886e2aeea98 100644 --- a/dali/dfuXRefLib/dfuxreflib.cpp +++ b/dali/dfuXRefLib/dfuxreflib.cpp @@ -339,7 +339,7 @@ static unsigned short getDafsPort(const SocketEndpoint &ep,unsigned &numfails,Cr if (nPort) return nPort; StringBuffer err("Failed to connect to DaFileSrv on "); - ep.getIpText(err); + ep.getHostText(err); #ifdef _WIN32 OERRLOG("%s",err.str()); if (sect) { @@ -565,7 +565,7 @@ class CXRefManagerBase ForEachItemIn(i,inprogress) { if (i) msg.append(", "); - inprogress.item(i).getIpText(msg); + inprogress.item(i).getHostText(msg); } if (msgcallback) msgcallback->progress(msg.str()); @@ -1096,7 +1096,7 @@ class COrphanEntry: public CInterface out.append(", "); const SocketEndpoint &item = epa.item(sorted[i2]); StringBuffer cur; - item.getUrlStr(cur); + item.getEndpointHostText(cur); const char *s1 = prefix.str(); const char *s2 = cur.str(); if (prefix.length()&&(memcmp(s1,s2,prefix.length())==0)) @@ -1186,11 +1186,11 @@ class COrphanEntry: public CInterface ForEachItemIn(i3,parts[i]) { unsigned p = parts[i].item(i3); if (copy) { - replicateepa.item(p).getUrlStr(buf.clear()); + replicateepa.item(p).getEndpointHostText(buf.clear()); b->addProp("RNode",buf.str()); } else { - primaryepa.item(p).getUrlStr(buf.clear()); + primaryepa.item(p).getEndpointHostText(buf.clear()); b->addProp("Node",buf.str()); } } @@ -1314,7 +1314,7 @@ IPropertyTree *CLogicalNameEntry::addFileBranch(IPropertyTree *dst,unsigned flag continue; IPropertyTree *part = addBranch(out,"Part"); if (flags&CFBpartnode) { - grp->queryNode((i+rep)%grp->ordinality()).endpoint().getUrlStr(buf.clear()); // TBD check grp==cluster TBD + grp->queryNode((i+rep)%grp->ordinality()).endpoint().getEndpointHostText(buf.clear()); // TBD check grp==cluster TBD part->setProp("Node",buf.str()); } if (flags&CFBpartnum) { @@ -1861,7 +1861,7 @@ class CXRefManager: public CXRefManagerBase try { StringBuffer msg; INode &node = g->queryNode(idx); - node.endpoint().getUrlStr(msg); + node.endpoint().getEndpointHostText(msg); manager.log("Getting directories for %s",msg.str()); manager.addNodeInProgress(node); Owned results; @@ -2298,10 +2298,10 @@ class CXRefManager: public CXRefManagerBase item.minsize = 0; StringBuffer s1; if (!item.minip.isNull()) - item.minip.getIpText(s1); + item.minip.getHostText(s1); StringBuffer s2; if (!item.maxip.isNull()) - item.maxip.getIpText(s2); + item.maxip.getHostText(s2); StringBuffer skew; item.getskew(skew); outf("%s numfiles=%u totalsize=%" CF "d minsize=%" CF "d(%s) maxsize=%" CF "d(%s), skew=%s\n",item.name.get(),item.num,item.size, @@ -2567,12 +2567,12 @@ class CXRefManager: public CXRefManagerBase t->addPropInt64("MaxSize",item.maxsize); StringBuffer s1; if (!item.maxip.isNull()) - item.maxip.getIpText(s1); + item.maxip.getHostText(s1); t->addProp("MaxIP",s1.str()); t->addPropInt64("MinSize",item.minsize); s1.clear(); if (!item.minip.isNull()) - item.minip.getIpText(s1); + item.minip.getHostText(s1); t->addProp("MinIP",s1.str()); item.getskew(s1.clear()); if (s1.length()) diff --git a/dali/dfuplus/dfuplus.cpp b/dali/dfuplus/dfuplus.cpp index 8b6b2697e3c..5623fb94903 100644 --- a/dali/dfuplus/dfuplus.cpp +++ b/dali/dfuplus/dfuplus.cpp @@ -68,7 +68,7 @@ class CDafsThread: public Thread if (listenep.isNull()) eps.append(listenep.port); else - listenep.getUrlStr(eps); + listenep.getEndpointHostText(eps); #endif server.setown(createRemoteFileServer()); server->setThrottle(ThrottleStd, 0); // disable throttling @@ -152,7 +152,7 @@ bool CDfuPlusHelper::runLocalDaFileSvr(SocketEndpoint &listenep,bool requireauth addlPort.appendf(":%u", sslport); } StringBuffer eps; - progress("Started local Dali file server on %s%s\n", printep.getUrlStr(eps).str(), addlPort.str()); + progress("Started local Dali file server on %s%s\n", printep.getEndpointHostText(eps).str(), addlPort.str()); } thr->start(); @@ -663,7 +663,7 @@ int CDfuPlusHelper::spray() { SocketEndpoint localep; if (checkLocalDaFileSvr(srcip,localep)) - srcip = localep.getUrlStr(localeps).str(); + srcip = localep.getEndpointHostText(localeps).str(); } StringBuffer wuid; StringBuffer errmsg; @@ -832,7 +832,7 @@ int CDfuPlusHelper::despray() SocketEndpoint localep; StringBuffer localeps; if (isEmptyString(dstplane) && checkLocalDaFileSvr(dstip,localep)) - dstip = localep.getUrlStr(localeps).str(); + dstip = localep.getEndpointHostText(localeps).str(); Owned result = sprayclient->Despray(req); const char* wuid = result->getWuid(); if(isEmptyString(wuid)) diff --git a/dali/ft/daftformat.cpp b/dali/ft/daftformat.cpp index 475b44114dd..02040383302 100644 --- a/dali/ft/daftformat.cpp +++ b/dali/ft/daftformat.cpp @@ -1889,7 +1889,7 @@ void CRemotePartitioner::callRemote() { LogMsgJobInfo job(unknownJob); StringBuffer url, tmp; - ep.getUrlStr(url); + ep.getEndpointHostText(url); MemoryBuffer msg; msg.setEndian(__BIG_ENDIAN); diff --git a/dali/ft/filecopy.cpp b/dali/ft/filecopy.cpp index 318ae13ab66..ed0f7d2cfc4 100644 --- a/dali/ft/filecopy.cpp +++ b/dali/ft/filecopy.cpp @@ -203,7 +203,7 @@ FileTransferThread::FileTransferThread(FileSprayer & _sprayer, byte _action, con if (isContainerized()) url.set(sprayer.sprayServiceHost); else - ep.getUrlStr(url); + ep.getEndpointHostText(url); // progressInfo = _progressInfo; sem = NULL; ok = false; @@ -1034,7 +1034,7 @@ void FileSprayer::beforeTransfer() Owned factory = getEnvironmentFactory(true); Owned env = factory->openEnvironment(); StringBuffer ipText; - targets.item(0).filename.queryIP().getIpText(ipText); + targets.item(0).filename.queryIP().getHostText(ipText); Owned machine = env->getMachineByAddress(ipText.str()); if (machine) { @@ -3786,7 +3786,7 @@ void FileSprayer::splitAndCollectFileInfo(IPropertyTree * newRecord, RemoteFilen if (drive.isEmpty()) { - remoteFileName.queryIP().getIpText(drive.clear()); + remoteFileName.queryIP().getHostText(drive.clear()); newRecord->setProp("@ip", drive.str()); } else @@ -3902,7 +3902,7 @@ bool FileSprayer::calcUsePull() const if (!sources.item(idx2).canPush()) { StringBuffer s; - sources.item(idx2).filename.queryIP().getIpText(s); + sources.item(idx2).filename.queryIP().getHostText(s); LOG(MCdebugInfo, job, "Use pull operation because %s cannot push", s.str()); return true; } @@ -3910,7 +3910,7 @@ bool FileSprayer::calcUsePull() const if (!canLocateSlaveForNode(sources.item(0).filename.queryIP())) { StringBuffer s; - sources.item(0).filename.queryIP().getIpText(s); + sources.item(0).filename.queryIP().getHostText(s); LOG(MCdebugInfo, job, "Use pull operation because %s doesn't appear to have an ftslave", s.str()); return true; } @@ -3920,7 +3920,7 @@ bool FileSprayer::calcUsePull() const if (!targets.item(idx).canPull()) { StringBuffer s; - targets.item(idx).queryIP().getIpText(s); + targets.item(idx).queryIP().getHostText(s); LOG(MCdebugInfo, job, "Use push operation because %s cannot pull", s.str()); return false; } @@ -3929,7 +3929,7 @@ bool FileSprayer::calcUsePull() const if (!canLocateSlaveForNode(targets.item(0).queryIP())) { StringBuffer s; - targets.item(0).queryIP().getIpText(s); + targets.item(0).queryIP().getHostText(s); LOG(MCdebugInfo, job, "Use push operation because %s doesn't appear to have an ftslave", s.str()); return false; } diff --git a/dali/ft/fttransform.cpp b/dali/ft/fttransform.cpp index 84f64c1b54f..ee3ab53a3ee 100644 --- a/dali/ft/fttransform.cpp +++ b/dali/ft/fttransform.cpp @@ -588,8 +588,8 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action) if (!isContainerized() && !ep.isLocal()) { StringBuffer host, expected; - queryHostIP().getIpText(host); - ep.getIpText(expected); + queryHostIP().getHostText(host); + ep.getHostText(expected); throwError2(DFTERR_WrongComputer, expected.str(), host.str()); } diff --git a/dali/sasha/packetstore.cpp b/dali/sasha/packetstore.cpp index ea22616d2e4..d20dd353689 100644 --- a/dali/sasha/packetstore.cpp +++ b/dali/sasha/packetstore.cpp @@ -416,7 +416,7 @@ class CPutTransactionQueue CriticalBlock block(sect); if (doremove(sender,transactionid)) { StringBuffer s; - IERRLOG("Warning: duplicate transaction detected from %s", sender.getUrlStr(s).str()); + IERRLOG("Warning: duplicate transaction detected from %s", sender.getEndpointHostText(s).str()); exit(0); } p->setTransactionId(sender,transactionid); diff --git a/dali/sasha/saserver.cpp b/dali/sasha/saserver.cpp index 6ede9f3556a..6d802e4fdef 100644 --- a/dali/sasha/saserver.cpp +++ b/dali/sasha/saserver.cpp @@ -443,7 +443,7 @@ int main(int argc, const char* argv[]) #endif StringBuffer eps; - PROGLOG("SASERVER starting on %s",queryMyNode()->endpoint().getUrlStr(eps).str()); + PROGLOG("SASERVER starting on %s",queryMyNode()->endpoint().getEndpointHostText(eps).str()); ForEachItemIn(i1,servers) { diff --git a/dali/sasha/sasha.cpp b/dali/sasha/sasha.cpp index 3f7b02de1ce..81b002463f3 100644 --- a/dali/sasha/sasha.cpp +++ b/dali/sasha/sasha.cpp @@ -379,7 +379,7 @@ bool getVersion(INode *node) Owned cmd = createSashaCommand(); cmd->setAction(SCA_GETVERSION); StringBuffer ips; - node->endpoint().getIpText(ips); + node->endpoint().getHostText(ips); if (!cmd->send(node,1*60*1000)) { OERRLOG("Could not connect to Sasha server on %s",ips.str()); return false; diff --git a/dali/sasha/saverify.cpp b/dali/sasha/saverify.cpp index f4c63460181..aec54d09e2c 100644 --- a/dali/sasha/saverify.cpp +++ b/dali/sasha/saverify.cpp @@ -252,7 +252,7 @@ class CFileCrcList SocketEndpoint ep(testpart->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); return; } @@ -276,7 +276,7 @@ class CFileCrcList SocketEndpoint ep(part->queryNode()->endpoint()); if (!dafilesrvips.verifyDaliFileServer(ep)) { StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); PROGLOG("VERIFY: file %s, cannot run DAFILESRV on %s",name,ips.str()); continue; } @@ -315,7 +315,7 @@ class CFileCrcList try { partfile.setown(createIFile(rfn)); - // PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + // PROGLOG("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); if (partfile) { if (parent->stopped) return; @@ -329,7 +329,7 @@ class CFileCrcList catch (IException *e) { StringBuffer s; - s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + s.appendf("VERIFY: part %s on %s",partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); EXCLOG(e, s.str()); e->Release(); ok = false; @@ -540,7 +540,7 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread if (rver==0) { StringBuffer epstr; SocketEndpoint ep = eps.item(i); - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); CriticalBlock block(sect); if (failurelimit) { LOG(MCoperatorError, unknownJob,"DAFSMON: dafilesrv on %s cannot be contacted",epstr.str()); @@ -554,7 +554,7 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread CriticalBlock block(sect); StringBuffer epstr; SocketEndpoint ep = eps.item(i); - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); StringBuffer s; s.appendf("DAFSMON: dafilesrv %s",epstr.str()); LOG(MCoperatorError, unknownJob, e, s.str()); @@ -599,7 +599,7 @@ class CSashaDaFSMonitorServer: public ISashaServer, public Thread if (!ep.isNull()) { if (trc.length()) trc.append(", "); - ep.getUrlStr(trc); + ep.getEndpointHostText(trc); if (ep.port==0) ep.port = getDaliServixPort(); eps.append(ep); diff --git a/dali/sasha/saxref.cpp b/dali/sasha/saxref.cpp index 76ed64fdaf0..a7ef0ce6685 100644 --- a/dali/sasha/saxref.cpp +++ b/dali/sasha/saxref.cpp @@ -800,7 +800,7 @@ class CNewXRefManager: public CNewXRefManagerBase } } // add the first IP also - rawgrp->queryNode(0).endpoint().getIpText(gname.clear()); + rawgrp->queryNode(0).endpoint().getHostText(gname.clear()); clusters.append(gname.str()); clusterscsl.append(',').append(gname.str()); if (basedir.length()==0) { @@ -970,7 +970,7 @@ class CNewXRefManager: public CNewXRefManagerBase StringBuffer path(rootdir); SocketEndpoint ep = parent.rawgrp->queryNode(i).endpoint(); StringBuffer tmp; - parent.log("Scanning %s directory %s",ep.getUrlStr(tmp).str(),path.str()); + parent.log("Scanning %s directory %s",ep.getEndpointHostText(tmp).str(),path.str()); if (!parent.scanDirectory(i,ep,path,0,NULL,NULL)) { ok = false; return; @@ -978,7 +978,7 @@ class CNewXRefManager: public CNewXRefManagerBase i = (i+r)%n; setReplicateFilename(path,1); ep = parent.rawgrp->queryNode(i).endpoint(); - parent.log("Scanning %s directory %s",ep.getUrlStr(tmp.clear()).str(),path.str()); + parent.log("Scanning %s directory %s",ep.getEndpointHostText(tmp.clear()).str(),path.str()); if (!parent.scanDirectory(i,ep,path,1,NULL,NULL)) { ok = false; } @@ -1179,7 +1179,7 @@ class CNewXRefManager: public CNewXRefManagerBase pb->setPropInt("Num",i); pb = branch->addPropTree("Part",pb); } - pb->setProp(rep?"RNode":"Node",ep.getUrlStr(tmp.clear()).str()); + pb->setProp(rep?"RNode":"Node",ep.getEndpointHostText(tmp.clear()).str()); } @@ -1396,12 +1396,12 @@ class CNewXRefManager: public CNewXRefManagerBase StringBuffer s1; if (d->maxnode[drv]) { dt->addPropInt64("MaxSize",d->maxsize[drv]); - grp->queryNode(d->maxnode[drv]-1).endpoint().getIpText(s1); + grp->queryNode(d->maxnode[drv]-1).endpoint().getHostText(s1); dt->addProp("MaxIP",s1.str()); } if (d->minnode[drv]) { dt->addPropInt64("MinSize",d->minsize[drv]); - grp->queryNode(d->minnode[drv]-1).endpoint().getIpText(s1.clear()); + grp->queryNode(d->minnode[drv]-1).endpoint().getHostText(s1.clear()); dt->addProp("MinIP",s1.str()); } if (d->minsize[drv]maxsize[drv]) { @@ -1597,7 +1597,7 @@ class CNewXRefManager: public CNewXRefManagerBase if (lost) { Owned pt = createPTree("Part"); StringBuffer tmp; - rfn.queryEndpoint().getIpText(tmp); + rfn.queryEndpoint().getHostText(tmp); pt->setProp("Node",tmp.str()); pt->setPropInt("Num",pn+1); if (copy>0) diff --git a/dali/server/daserver.cpp b/dali/server/daserver.cpp index 89c82708b85..d02451e3ca6 100644 --- a/dali/server/daserver.cpp +++ b/dali/server/daserver.cpp @@ -217,7 +217,7 @@ static bool populateAllowListFromEnvironment(IAllowListWriter &writer) IpAddress ip(it->second.c_str()); if (ip.isNull()) return defaultValue; - return ip.getIpText(result); + return ip.getHostText(result); }; auto addRoles = [&writer, &resolveComputer](const IPropertyTree &component, const std::initializer_list &roles) { @@ -361,7 +361,7 @@ static bool populateAllowListFromEnvironment(IAllowListWriter &writer) if (!ip.isNull()) { StringBuffer ipStr; - ip.getIpText(ipStr); + ip.getHostText(ipStr); ForEachItemIn(r, roles) { const char *roleStr = roles.item(r); diff --git a/deployment/configenv/xml_jlibpt/Hardware.cpp b/deployment/configenv/xml_jlibpt/Hardware.cpp index beda50e11f2..f9bf92fdfc5 100644 --- a/deployment/configenv/xml_jlibpt/Hardware.cpp +++ b/deployment/configenv/xml_jlibpt/Hardware.cpp @@ -117,7 +117,7 @@ IPropertyTree* Hardware::addComputer(IPropertyTree *params) else { ipAddr.ipset(computerName); - ipAddr.getIpText(sbIp); + ipAddr.getHostText(sbIp); } } else diff --git a/deployment/configgen/main.cpp b/deployment/configgen/main.cpp index c4fbb739d30..bb91c0dbfd7 100644 --- a/deployment/configgen/main.cpp +++ b/deployment/configgen/main.cpp @@ -206,7 +206,7 @@ void getTempPath(char* tempPath, unsigned int bufsize, const char* subdir/*=NULL void replaceDotWithHostIp(IPropertyTree* pTree, bool verbose) { StringBuffer ip; - queryHostIP().getIpText(ip); + queryHostIP().getHostText(ip); const char* attrs[] = {"@netAddress", "@roxieAddress", "@daliAddress"}; StringBuffer xPath; @@ -827,7 +827,7 @@ int main(int argc, char** argv) if (ip.isLoopBack()) // assume they meant any local ip... not sure this is a good idea ipAddr.append('.'); else - ip.getIpText(ipAddr); + ip.getHostText(ipAddr); } } else if(stricmp(argv[i], "-od") == 0) diff --git a/deployment/deploy/DeployTask.cpp b/deployment/deploy/DeployTask.cpp index 801b806f361..841cab4edc3 100644 --- a/deployment/deploy/DeployTask.cpp +++ b/deployment/deploy/DeployTask.cpp @@ -1696,7 +1696,7 @@ class CDeployTask : public CInterface, implements IDeployTask throw MakeStringException(-1, "Cannot resolve %s", remoteNameOrIp); StringBuffer remoteIP; - ip.getIpText(remoteIP); + ip.getHostText(remoteIP); HANDLE hEnum; @@ -1734,7 +1734,7 @@ class CDeployTask : public CInterface, implements IDeployTask } StringBuffer ipAddr; - ip.getIpText(ipAddr); + ip.getHostText(ipAddr); if (!stricmp(remoteIP.str(), ipAddr.str())) { diff --git a/deployment/deployutils/deployutils.cpp b/deployment/deployutils/deployutils.cpp index 683e56db6d6..4a717f1ac28 100644 --- a/deployment/deployutils/deployutils.cpp +++ b/deployment/deployutils/deployutils.cpp @@ -3078,7 +3078,7 @@ IPropertyTree* getNewRange(const IPropertyTree* pEnv, const char* prefix, const while (count--) { - range.getIpText(sIP.clear()); + range.getHostText(sIP.clear()); unsigned x; range.getNetAddress(sizeof(x),&x); diff --git a/deployment/envgen/main.cpp b/deployment/envgen/main.cpp index fe7d52bd982..a8331c67c0f 100644 --- a/deployment/envgen/main.cpp +++ b/deployment/envgen/main.cpp @@ -391,7 +391,7 @@ int main(int argc, char** argv) StringBuffer env; StringBuffer thisip; - queryHostIP().getIpText(thisip); + queryHostIP().getHostText(thisip); env.appendf("<" XML_HEADER ">\n\n", thisip.str()); env.append(envXml); diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index 0cd042d5102..ad833bed776 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -11,6 +11,7 @@ GITHUB_REF=$(git rev-parse --short=8 HEAD) cd vcpkg VCPKG_REF=$(git rev-parse --short=8 HEAD) cd .. +GITHUB_BRANCH=$(git log -50 --pretty=format:"%D" | tr ',' '\n' | grep 'upstream/' | awk 'NR==1 {sub("upstream/", ""); print}') DOCKER_USERNAME="${DOCKER_USERNAME:-hpccbuilds}" DOCKER_PASSWORD="${DOCKER_PASSWORD:-none}" @@ -19,34 +20,42 @@ echo "GITHUB_ACTOR: $GITHUB_ACTOR" echo "GITHUB_TOKEN: $GITHUB_TOKEN" echo "GITHUB_REF: $GITHUB_REF" echo "VCPKG_REF: $VCPKG_REF" +echo "GITHUB_BRANCH: $GITHUB_BRANCH" echo "DOCKER_USERNAME: $DOCKER_USERNAME" echo "DOCKER_PASSWORD: $DOCKER_PASSWORD" -# docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD +docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD + +CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" function doBuild() { docker build --progress plain --pull --rm -f "$SCRIPT_DIR/$1.dockerfile" \ - -t build-$1:$GITHUB_REF \ - -t build-$1:latest \ --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \ --build-arg VCPKG_REF=$VCPKG_REF \ - "$SCRIPT_DIR/." + -t hpccsystems/platform-build-$1:$VCPKG_REF \ + -t hpccsystems/platform-build-$1:$GITHUB_BRANCH \ + --cache-from hpccsystems/platform-build-$1:$VCPKG_REF \ + --cache-from hpccsystems/platform-build-$1:$GITHUB_BRANCH \ + "$SCRIPT_DIR/." + + docker push hpccsystems/platform-build-$1:$VCPKG_REF & + docker push hpccsystems/platform-build-$1:$GITHUB_BRANCH & - docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \ + docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached hpccsystems/platform-build-$1:$VCPKG_REF \ "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS} && \ - cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel $(nproc)" + cmake --build /hpcc-dev/HPCC-Platform/build-$1 --target package --parallel $(nproc)" # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash } -CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" - -doBuild centos-7 -doBuild centos-8 -doBuild amazonlinux -doBuild ubuntu-23.04 -doBuild ubuntu-22.04 +doBuild ubuntu-23.04 doBuild ubuntu-20.04 +doBuild amazonlinux +doBuild ubuntu-22.04 +doBuild centos-8 +doBuild centos-7 + +wait # docker build --progress plain --pull --rm -f "$SCRIPT_DIR/core.dockerfile" \ # -t $DOCKER_USERNAME/core:$GITHUB_REF \ diff --git a/ecl/eclagent/eclagent.cpp b/ecl/eclagent/eclagent.cpp index 26d86be7056..1ca18ca42c4 100644 --- a/ecl/eclagent/eclagent.cpp +++ b/ecl/eclagent/eclagent.cpp @@ -378,7 +378,7 @@ class CHThorDebugSocketWorker : public CInterface, implements IPooledThread StringBuffer sanitizedText; StringAttr queryName; StringBuffer peerStr; - peer.getIpText(peerStr); + peer.getHostText(peerStr); const char *uid = "-"; StringBuffer ctxstr; try @@ -1917,7 +1917,7 @@ void EclAgent::doProcess() w->setDebugAgentListenerPort(debugContext->queryPort()); StringBuffer sb; - queryHostIP().getIpText(sb); + queryHostIP().getHostText(sb); w->setDebugAgentListenerIP(sb); } if(resetWorkflow) @@ -3341,7 +3341,7 @@ char * EclAgent::getDaliServers() first = false; else dali.append(','); - coven->query().endpoint().getUrlStr(dali); + coven->query().endpoint().getEndpointHostText(dali); } return dali.detach(); } diff --git a/ecl/eclccserver/eclccserver.cpp b/ecl/eclccserver/eclccserver.cpp index 7426ee8c197..858eb2dc67d 100644 --- a/ecl/eclccserver/eclccserver.cpp +++ b/ecl/eclccserver/eclccserver.cpp @@ -831,7 +831,9 @@ class EclccCompileThread : implements IPooledThread, implements IErrorReporter, } else { - getHomeFolder(repoRootPath); + char dir[_MAX_PATH]; + if (GetCurrentDirectory(sizeof(dir), dir)) + repoRootPath.append(dir); } if (repoRootPath.length()) { diff --git a/ecl/eclcmd/eclcmd_core.cpp b/ecl/eclcmd/eclcmd_core.cpp index 8f3922dba42..62ad2dd01c5 100644 --- a/ecl/eclcmd/eclcmd_core.cpp +++ b/ecl/eclcmd/eclcmd_core.cpp @@ -18,6 +18,7 @@ #include #include "jlog.hpp" #include "jfile.hpp" +#include "jsecrets.hpp" #include "jargv.hpp" #include "jflz.hpp" #include "httpclient.hpp" @@ -2115,6 +2116,90 @@ class EclCmdZapGen : public EclCmdCommon }; + +class EclCmdUrlMapSecretName : public CInterfaceOf +{ +public: + EclCmdUrlMapSecretName() + { + + } + + virtual eclCmdOptionMatchIndicator parseCommandLineOptions(ArgvIterator &iter) override + { + eclCmdOptionMatchIndicator retVal = EclCmdOptionNoMatch; + if (iter.done()) + return EclCmdOptionNoMatch; + + for (; !iter.done(); iter.next()) + { + const char *arg = iter.query(); + if (*arg != '-') //parameters don't start with '-' + { + if (optUrl.length()) + { + fprintf(stderr, "\nunrecognized argument %s\n", arg); + return EclCmdOptionCompletion; + } + optUrl.set(arg); + retVal = EclCmdOptionMatch; + continue; + } + if (iter.matchOption(optUsername, ECLOPT_USERNAME)) + { + retVal = EclCmdOptionMatch; + continue; + } + } + return retVal; + } + virtual bool finalizeOptions(IProperties *globals) override + { + if (optUrl.isEmpty()) + { + fprintf(stdout, "\n URL parameter required.\n"); + return false; + } + return true; + } + virtual int processCMD() override + { + StringBuffer secretName; + generateDynamicUrlSecretName(secretName, optUrl, optUsername); + if (secretName.isEmpty()) + { + fputs("Error genenerating secret name.", stderr); + return 1; + } + fputs(secretName.str(), stdout); + fputs("\n", stdout); + return 0; + } + virtual void usage() override + { + fputs("\nUsage:\n" + "\n" + "The 'url-secret-name' command generates a secret name from a url that can be used to support\n" + " ECL SOAPCALL/HTTPCALL automated url to secret mapping.\n" + " Username can either be embedded in the url, such as https://username@example.com, or\n" + " Passed in as a parameter --username=username\n" + " Passwords embedded in the URL are not needed and will be ignored.\n" + "\n" + "When ECL SOAPCALL URL secret mapping is enabled SOAPCALL will convert the URL provided into a name of this format.\n" + " ECL will then attempt to lookup the secret, and if found will use the contents of the secret, rather then the original url.\n" + "\n" + "ecl url-secret-name [--username=]\n" + "\n" + " URL the URL to convert into a secret name\n" + " Options:\n" + " --username Username to associate with the URL. Will override any username embedded in the URL.\n", + stdout); + } +private: + StringAttr optUrl; + StringAttr optUsername; +}; + //========================================================================================= IEclCommand *createCoreEclCommand(const char *cmdname) @@ -2145,6 +2230,8 @@ IEclCommand *createCoreEclCommand(const char *cmdname) return new EclCmdStatus(); if (strieq(cmdname, "zapgen")) return new EclCmdZapGen(); + if (strieq(cmdname, "url-secret-name")) + return new EclCmdUrlMapSecretName(); if (strieq(cmdname, "sign")) return createSignEclCommand(); if (strieq(cmdname, "listkeyuid")) diff --git a/ecl/wutest/wujobqtest.cpp b/ecl/wutest/wujobqtest.cpp index 2e748cd36fe..276f8d0f07f 100644 --- a/ecl/wutest/wujobqtest.cpp +++ b/ecl/wutest/wujobqtest.cpp @@ -155,11 +155,11 @@ void testEnqueue(unsigned nthreads,const char *qname) { unsigned start = msTick(); if (!partfile->remove()&&(copy==0)) // only warn about missing primary files - LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getUrlStr(eps).str()); + LOG(MCwarning, unknownJob, "Failed to remove file part %s from %s", partfile->queryFilename(),rfn.queryEndpoint().getEndpointHostText(eps).str()); else { unsigned t = msTick()-start; if (t>5*1000) - LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getUrlStr(eps).str(), t/1000); + LOG(MCwarning, unknownJob, "Removing %s from %s took %ds", partfile->queryFilename(), rfn.queryEndpoint().getEndpointHostText(eps).str(), t/1000); } } @@ -171,7 +171,7 @@ void testEnqueue(unsigned nthreads,const char *qname) else { StringBuffer s("Failed to remove file part "); s.append(partfile->queryFilename()).append(" from "); - rfn.queryEndpoint().getUrlStr(s); + rfn.queryEndpoint().getUrgetEndpointHostTextlStr(s); EXCLOG(e, s.str()); e->Release(); } @@ -258,7 +258,7 @@ void testEnqueue(unsigned nthreads,const char *qname) myep.serialize(msg); // only used for tracing if (!conversation->send(msg)) { StringBuffer s("Failed to send query to Thor on "); - thorMaster.getUrlStr(s); + thorMaster.getEndpointHostText(s); throw MakeStringException(-1, s.str()); // maybe retry? } @@ -275,7 +275,7 @@ static void cmd_list(IJobQueue *queue) IJobQueueItem &item = iter->query(); StringBuffer eps; StringBuffer dts; - printf("%3d: %s owner=%s priority=%d session=%" I64F "x ep=%s port=%d enqueuedt=%s\n",n,item.queryWUID(),item.queryOwner(),item.getPriority(),item.getSessionId(),item.queryEndpoint().getUrlStr(eps).str(),item.getPort(),item.queryEnqueuedTime().getString(dts).str()); + printf("%3d: %s owner=%s priority=%d session=%" I64F "x ep=%s port=%d enqueuedt=%s\n",n,item.queryWUID(),item.queryOwner(),item.getPriority(),item.getSessionId(),item.queryEndpoint().getEndpointHostText(eps).str(),item.getPort(),item.queryEnqueuedTime().getString(dts).str()); } } @@ -420,7 +420,7 @@ static void cmd_dequeue(IJobQueue *queue) return; } StringBuffer eps; - printf("%s owner=%s priority=%d session=%" I64F "x ep=%s port=%d\n",item->queryWUID(),item->queryOwner(),item->getPriority(),item->getSessionId(),item->queryEndpoint().getUrlStr(eps).str(),item->getPort()); + printf("%s owner=%s priority=%d session=%" I64F "x ep=%s port=%d\n",item->queryWUID(),item->queryOwner(),item->getPriority(),item->getSessionId(),item->queryEndpoint().getEndpointHostText(eps).str(),item->getPort()); queue->disconnect(); } diff --git a/esp/bindings/http/client/httpclient.cpp b/esp/bindings/http/client/httpclient.cpp index 5cb3f38aeaa..6f9968ea850 100644 --- a/esp/bindings/http/client/httpclient.cpp +++ b/esp/bindings/http/client/httpclient.cpp @@ -329,7 +329,7 @@ int CHttpClient::connect(StringBuffer& errmsg, bool forceNewConnection) catch(IException *e) { StringBuffer url; - UERRLOG("Error connecting to %s", ep.getUrlStr(url).str()); + UERRLOG("Error connecting to %s", ep.getEndpointHostText(url).str()); DBGLOG(e); e->Release(); m_socket = nullptr; @@ -338,7 +338,7 @@ int CHttpClient::connect(StringBuffer& errmsg, bool forceNewConnection) catch(...) { StringBuffer url; - UERRLOG("Unknown exception connecting to %s", ep.getUrlStr(url).str()); + UERRLOG("Unknown exception connecting to %s", ep.getEndpointHostText(url).str()); m_socket = nullptr; return -1; } @@ -346,7 +346,7 @@ int CHttpClient::connect(StringBuffer& errmsg, bool forceNewConnection) if(m_socket == nullptr) { StringBuffer urlstr; - DBGLOG(">>Can't connect to %s", ep.getUrlStr(urlstr).str()); + DBGLOG(">>Can't connect to %s", ep.getEndpointHostText(urlstr).str()); return -1; } @@ -484,7 +484,7 @@ bool appendProxyPeerAddress(CHttpMessage ©To, ISocket *sock, const char *src StringBuffer s(src); if (s.length()) s.append(", "); - ip.getIpText(s); + ip.getHostText(s); copyTo.setHeader("X-Forwarded-For", s); return true; } diff --git a/esp/bindings/http/platform/httpbinding.cpp b/esp/bindings/http/platform/httpbinding.cpp index 43cc62f34ad..a659999d28e 100644 --- a/esp/bindings/http/platform/httpbinding.cpp +++ b/esp/bindings/http/platform/httpbinding.cpp @@ -369,7 +369,7 @@ EspHttpBinding::EspHttpBinding(IPropertyTree* tree, const char *bindname, const { const char *host=getHost(); if (!host || !(*host) || !strcmp(host, ".") || !strcmp(host, "0.0.0.0")) - queryHostIP().getIpText(m_wsdlAddress); + queryHostIP().getHostText(m_wsdlAddress); else m_wsdlAddress.append(host); if (!strchr(m_wsdlAddress.str(), ':') && m_port!=80 && m_port!=443) diff --git a/esp/clients/roxiecontrol.cpp b/esp/clients/roxiecontrol.cpp index ab984873e09..6f772f431b7 100644 --- a/esp/clients/roxiecontrol.cpp +++ b/esp/clients/roxiecontrol.cpp @@ -142,7 +142,7 @@ static ISocket *createRoxieControlSocket(ISmartSocketFactory *conn, unsigned wai { StringBuffer err; err.append("Failure to establish secure connection to "); - ep.getUrlStr(err); + ep.getEndpointHostText(err); err.append(": returned ").append(status); throw makeStringException(SECURE_CONNECTION_FAILURE, err.str()); } diff --git a/esp/clients/wsdfuaccess/wsdfuaccess.cpp b/esp/clients/wsdfuaccess/wsdfuaccess.cpp index 8a585bc7a75..7850464c80f 100644 --- a/esp/clients/wsdfuaccess/wsdfuaccess.cpp +++ b/esp/clients/wsdfuaccess/wsdfuaccess.cpp @@ -543,7 +543,7 @@ StringBuffer &encodeDFUFileMeta(StringBuffer &metaInfoBlob, IPropertyTree *metaI * If the size of this initial request was ever a concern, we could consider other ways to ensure a one-off * delivery of this esp public signing cert. to dafilesrv, e.g. by dafilesrv reaching out to esp to request it. */ - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "encodeDFUFileMeta: No '%s' MTLS certificate detected.", keyPairName); privateKeyFName = info->queryProp("privatekey"); @@ -639,7 +639,7 @@ class DFUAccessTests : public CppUnit::TestFixture basePath.append("//"); SocketEndpoint ep(serverPort); - ep.getUrlStr(basePath); + ep.getEndpointHostText(basePath); char cpath[_MAX_DIR]; if (!GetCurrentDirectory(_MAX_DIR, cpath)) diff --git a/esp/esdlscriptlib/esdl_script.cpp b/esp/esdlscriptlib/esdl_script.cpp index b1a3db77ac8..979982bcfba 100644 --- a/esp/esdlscriptlib/esdl_script.cpp +++ b/esp/esdlscriptlib/esdl_script.cpp @@ -929,9 +929,7 @@ class CEsdlTransformOperationMySqlCall : public CEsdlTransformOperationBase StringBuffer vault; if (m_vaultName) sourceContext->evaluateAsString(m_vaultName, vault); - if (vault.isEmpty()) - return getSecret("esp", name); - return getVaultSecret("esp", vault, name); + return getSecret("esp", name, vault); } void appendOption(StringBuffer &options, const char *name, const char *value, bool required) { diff --git a/esp/platform/espp.hpp b/esp/platform/espp.hpp index 41f4dbf669a..ffc3682b455 100644 --- a/esp/platform/espp.hpp +++ b/esp/platform/espp.hpp @@ -203,7 +203,7 @@ class CEspServer : public CInterface, if (host != NULL) strIP.append(host); else - m_address.getIpText(strIP); + m_address.getHostText(strIP); LOG(MCprogress, "binding %s, on %s:%d", name, strIP.str(), port); diff --git a/esp/services/WsDeploy/WsDeployService.hpp b/esp/services/WsDeploy/WsDeployService.hpp index 57558f39747..b4f480f17f5 100644 --- a/esp/services/WsDeploy/WsDeployService.hpp +++ b/esp/services/WsDeploy/WsDeployService.hpp @@ -662,7 +662,7 @@ class CCloudActionHandler : public CInterface, implements IInterface Owned iter = pComputers->getElements(XML_TAG_COMPUTER); StringBuffer localip; - queryHostIP().getIpText(localip); + queryHostIP().getHostText(localip); ForEach(*iter) { @@ -775,7 +775,7 @@ class CCloudActionHandler : public CInterface, implements IInterface const char* getUser() {return m_user.str();} const char* getNewEnv() {return m_newEnv.str();} const char* getNewEnvId() {return m_newEnvId.str();} - const char* getCurIp(){ if (m_curIp.length() == 0) queryHostIP().getIpText(m_curIp); return m_curIp.str(); } + const char* getCurIp(){ if (m_curIp.length() == 0) queryHostIP().getHostText(m_curIp); return m_curIp.str(); } private: CWsDeployFileInfo* m_pFileInfo; diff --git a/esp/services/ws_dfu/ws_dfuService.cpp b/esp/services/ws_dfu/ws_dfuService.cpp index 4dcfd29b947..c7d13c8c3a7 100644 --- a/esp/services/ws_dfu/ws_dfuService.cpp +++ b/esp/services/ws_dfu/ws_dfuService.cpp @@ -2090,7 +2090,7 @@ void CWsDfuEx::getFilePartsOnClusters(IEspContext &context, const char *clusterR for (unsigned i=0; iendpoint().getUrlStr(url); + part.queryNode(i)->endpoint().getEndpointHostText(url); Owned FilePart = createDFUPart(); FilePart->setId(partIndex+1); @@ -2538,7 +2538,7 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co Owned FilePart = createDFUPart(); StringBuffer url; - part->queryNode(i)->endpoint().getUrlStr(url); + part->queryNode(i)->endpoint().getEndpointHostText(url); FilePart->setId(part->getPartIndex()+1); FilePart->setCopy(i+1); @@ -5970,7 +5970,7 @@ void CWsDfuEx::getFilePartsInfo(IEspContext &context, const char *dafilesrvHost, * In bare metal the host names from the group will be used. */ if (!dafilesrvHost) - part.queryNode(i)->endpoint().getUrlStr(host.clear()); + part.queryNode(i)->endpoint().getEndpointHostText(host.clear()); unsigned *locationIndex = partLocationMap.getValue(host.str()); if (locationIndex) fileCopy->setLocationIndex(*locationIndex); @@ -6112,7 +6112,7 @@ void CWsDfuEx::dFUFileAccessCommon(IEspContext &context, const CDfsLogicalFileNa StringBuffer dafilesrvHost; #ifdef _CONTAINERIZED keyPairName.set("signing"); - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "dFUFileAccessCommon: file signing certificate ('%s') not defined in configuration.", keyPairName.str()); @@ -6489,7 +6489,7 @@ bool CWsDfuEx::onDFUFileCreateV2(IEspContext &context, IEspDFUFileCreateV2Reques #ifdef _CONTAINERIZED keyPairName.set("signing"); - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "onDFUFileCreateV2: file signing certificate ('%s' ) not defined in configuration.", keyPairName.str()); diff --git a/esp/services/ws_ecl/ws_ecl_service.cpp b/esp/services/ws_ecl/ws_ecl_service.cpp index c2a087e0fe8..af2a2830117 100644 --- a/esp/services/ws_ecl/ws_ecl_service.cpp +++ b/esp/services/ws_ecl/ws_ecl_service.cpp @@ -2088,10 +2088,10 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri throw MakeStringException(-1, "roxie target cluster not mapped: %s", target); ep = conn->nextEndpoint(); - Owned httpctx = getHttpClientContext(); WsEclSocketFactory *roxieConn = static_cast(conn); + Owned httpctx = getHttpClientSecretContext(roxieConn->queryTlsIssuer()); StringBuffer url(roxieConn->isTlsService() ? "https://" : "http://"); - ep.getIpText(url).append(':').append(ep.port ? ep.port : 9876).append('/'); + ep.getHostText(url).append(':').append(ep.port ? ep.port : 9876).append('/'); if (roxieConn->includeTargetInURL) url.append(roxieConn->alias.isEmpty() ? target : roxieConn->alias.str()); if (!trim) diff --git a/esp/services/ws_fs/ws_fsBinding.cpp b/esp/services/ws_fs/ws_fsBinding.cpp index 353c591f245..6693bd96cf6 100644 --- a/esp/services/ws_fs/ws_fsBinding.cpp +++ b/esp/services/ws_fs/ws_fsBinding.cpp @@ -341,7 +341,7 @@ void CFileSpraySoapBindingEx::appendDropZones(double clientVersion, IConstEnviro IpAddress ipAddr; ipAddr.ipset(server.str()); - ipAddr.getIpText(networkAddress); + ipAddr.getHostText(networkAddress); if (!ipAddr.isNull()) { dropZone->addProp("@netAddress", networkAddress); diff --git a/esp/services/ws_fs/ws_fsService.cpp b/esp/services/ws_fs/ws_fsService.cpp index 7fabceabbd2..c46857a3f77 100644 --- a/esp/services/ws_fs/ws_fsService.cpp +++ b/esp/services/ws_fs/ws_fsService.cpp @@ -382,7 +382,7 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork SocketEndpoint srcdali; StringBuffer srcdaliip; file->getForeignDali(srcdali); - srcdali.getIpText(srcdaliip); + srcdali.getHostText(srcdaliip); if(srcdaliip.length() > 0 && strcmp(srcdaliip.str(), "0.0.0.0") != 0) dest.setSourceDali(srcdaliip.str()); StringBuffer diffkeyname; @@ -404,7 +404,7 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork Owned node = info->getNode(0); if (node) { - node->endpoint().getIpText(socket); + node->endpoint().getHostText(socket); dest.setSourceIP(socket.str()); } const char *defaultdir = info->queryDefaultDir(); @@ -484,7 +484,7 @@ static void DeepAssign(IEspContext &context, IConstDFUWorkUnit *src, IEspDFUWork Owned node = info->getNode(0); if (node) { - node->endpoint().getIpText(socket); + node->endpoint().getHostText(socket); dest.setDestIP(socket.str()); } const char *defaultdir = info->queryDefaultDir(); @@ -804,7 +804,7 @@ bool CFileSprayEx::GetArchivedDFUWorkunits(IEspContext &context, IEspGetDFUWorku StringBuffer url; throw MakeStringException(ECLWATCH_CANNOT_CONNECT_ARCHIVE_SERVER, "Sasha (%s) took too long to respond from: Get archived workUnits.", - ep.getUrlStr(url).str()); + ep.getEndpointHostText(url).str()); } IArrayOf results; @@ -1293,7 +1293,7 @@ void CFileSprayEx::getInfoFromSasha(IEspContext &context, const char *sashaServe StringBuffer url; throw MakeStringException(ECLWATCH_CANNOT_CONNECT_ARCHIVE_SERVER, "Sasha (%s) took too long to respond from: Get information for %s.", - ep.getUrlStr(url).str(), wuid); + ep.getEndpointHostText(url).str(), wuid); } if (cmd->numIds()==0) { diff --git a/esp/services/ws_machine/ws_machineService.cpp b/esp/services/ws_machine/ws_machineService.cpp index fccd5988089..212071d2d8f 100644 --- a/esp/services/ws_machine/ws_machineService.cpp +++ b/esp/services/ws_machine/ws_machineService.cpp @@ -815,7 +815,7 @@ void Cws_machineEx::getThorProcesses(IConstEnvironment* constEnv, IPropertyTree* ForEach(*gi) { StringBuffer addressRead; - gi->query().endpoint().getIpText(addressRead); + gi->query().endpoint().getHostText(addressRead); if (addressRead.length() == 0) { OWARNLOG("Network address not found for a node in node group %s", groupName.str()); @@ -831,7 +831,7 @@ void Cws_machineEx::getThorProcesses(IConstEnvironment* constEnv, IPropertyTree* else { IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(netAddress); + ipaddr.getHostText(netAddress); } if (netAddress.length() == 0) diff --git a/esp/services/ws_machine/ws_machineServiceMetrics.cpp b/esp/services/ws_machine/ws_machineServiceMetrics.cpp index 4e87b819423..bb09a586229 100644 --- a/esp/services/ws_machine/ws_machineServiceMetrics.cpp +++ b/esp/services/ws_machine/ws_machineServiceMetrics.cpp @@ -518,7 +518,7 @@ bool Cws_machineEx::onGetMetrics(IEspContext &context, IEspMetricsRequest &req, { IpAddress ip; ip.setNetAddress(sizeof(unsigned),lptr++); - ip.getIpText(ipBuf.clear()); + ip.getHostText(ipBuf.clear()); CMetricsThreadParam* pThreadReq = new CMetricsThreadParam(ipBuf.str(), req.getSecurityString(), @@ -620,7 +620,7 @@ bool Cws_machineEx::onGetMetrics(IEspContext &context, IEspMetricsRequest &req, { IpAddress ip; ip.setNetAddress(sizeof(unsigned),lptr++); - ip.getIpText(ipBuf.clear()); + ip.getHostText(ipBuf.clear()); ipList.append(ipBuf.str()); } diff --git a/esp/services/ws_machine/ws_machineServiceRexec.cpp b/esp/services/ws_machine/ws_machineServiceRexec.cpp index 32dc90c315f..adfef17ba57 100644 --- a/esp/services/ws_machine/ws_machineServiceRexec.cpp +++ b/esp/services/ws_machine/ws_machineServiceRexec.cpp @@ -275,7 +275,7 @@ class CStartStopThreadParam : public CRemoteExecThreadParam { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(ipStr); + ipaddr.getHostText(ipStr); if (ipStr.length() > 0) { #ifdef MACHINE_IP @@ -473,7 +473,7 @@ void Cws_machineEx::ConvertAddress( const char* originalAddress, StringBuffer& n { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(ipStr); + ipaddr.getHostText(ipStr); if (ipStr.length() > 0) { #ifdef MACHINE_IP diff --git a/esp/services/ws_smc/ws_smcService.cpp b/esp/services/ws_smc/ws_smcService.cpp index 78a8247bb32..e7b35f99a4b 100644 --- a/esp/services/ws_smc/ws_smcService.cpp +++ b/esp/services/ws_smc/ws_smcService.cpp @@ -1933,7 +1933,7 @@ bool CWsSMCEx::onBrowseResources(IEspContext &context, IEspBrowseResourcesReques { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(ipStr); + ipaddr.getHostText(ipStr); if (!ipStr.isEmpty()) resp.setNetAddress(ipStr.str()); } diff --git a/esp/services/ws_topology/ws_topologyService.cpp b/esp/services/ws_topology/ws_topologyService.cpp index ec1eb4a8b42..8cea7ff5959 100644 --- a/esp/services/ws_topology/ws_topologyService.cpp +++ b/esp/services/ws_topology/ws_topologyService.cpp @@ -1725,7 +1725,7 @@ bool CWsTopologyEx::onTpGetComponentFile(IEspContext &context, IEspTpGetComponen { StringBuffer ipStr; IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(ipStr); + ipaddr.getHostText(ipStr); if (ipStr.length() > 0) { netAddressStr = ipStr.str(); diff --git a/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp b/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp index a0893fa9396..e93bbf476e3 100644 --- a/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp +++ b/esp/services/ws_workunits/ws_workunitsAuditLogs.cpp @@ -1419,7 +1419,7 @@ void CWsWorkunitsSoapBindingEx::createAndDownloadWUZAPFile(IEspContext& context, if (zapInfoReq.esp.isEmpty()) { IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(zapInfoReq.esp); + ipaddr.getHostText(zapInfoReq.esp); } request->getParameter("ThorIPAddress", zapInfoReq.thor); } diff --git a/esp/services/ws_workunits/ws_workunitsHelpers.cpp b/esp/services/ws_workunits/ws_workunitsHelpers.cpp index 6c80b0a0e11..9332bf2a911 100644 --- a/esp/services/ws_workunits/ws_workunitsHelpers.cpp +++ b/esp/services/ws_workunits/ws_workunitsHelpers.cpp @@ -2370,7 +2370,7 @@ void WsWuInfo::getWorkunitThorSlaveLog(IGroup *nodeGroup, const char *ipAddress, StringBuffer slaveIPAddress; if (slaveNum > 0) { - nodeGroup->queryNode(slaveNum-1).endpoint().getIpText(slaveIPAddress); + nodeGroup->queryNode(slaveNum-1).endpoint().getHostText(slaveIPAddress); if (slaveIPAddress.length() < 1) throw makeStringException(ECLWATCH_INVALID_INPUT, "ThorSlave log network address not found."); diff --git a/esp/services/ws_workunits/ws_workunitsQuerySets.cpp b/esp/services/ws_workunits/ws_workunitsQuerySets.cpp index a0e63f4be54..6dd70d3700c 100644 --- a/esp/services/ws_workunits/ws_workunitsQuerySets.cpp +++ b/esp/services/ws_workunits/ws_workunitsQuerySets.cpp @@ -126,7 +126,7 @@ void fetchRemoteWorkunit(IClientWsWorkunits *_ws, IEspContext *ctx, const char * SocketEndpoint ep; checkUseEspOrDaliIP(ep, resp->getDaliServer(), netAddress); if (!ep.isNull()) - ep.getUrlStr(daliServer); + ep.getEndpointHostText(daliServer); } void fetchRemoteWorkunitAndQueryDetails(IClientWsWorkunits *_ws, IEspContext *ctx, const char *netAddress, const char *queryset, const char *query, const char *wuid, StringBuffer &name, StringBuffer &xml, StringBuffer &dllname, MemoryBuffer &dll, StringBuffer &daliServer, Owned &respQueryInfo, bool useSSL) diff --git a/esp/services/ws_workunits/ws_workunitsService.cpp b/esp/services/ws_workunits/ws_workunitsService.cpp index 2f1fc09c8df..6af6c3d23fd 100644 --- a/esp/services/ws_workunits/ws_workunitsService.cpp +++ b/esp/services/ws_workunits/ws_workunitsService.cpp @@ -128,7 +128,7 @@ IPropertyTree *getArchivedWorkUnitProperties(const char *wuid, bool dfuWU) if (!cmd->send(node, 1*60*1000)) throw MakeStringException(ECLWATCH_CANNOT_CONNECT_ARCHIVE_SERVER, "Sasha (%s) took too long to respond from: Get workUnit properties for %s.", - ep.getUrlStr(tmp).str(), wuid); + ep.getEndpointHostText(tmp).str(), wuid); if ((cmd->numIds() < 1) || (cmd->numResults() < 1)) return nullptr; @@ -400,7 +400,7 @@ void CWsWorkunitsEx::init(IPropertyTree *cfg, const char *process, const char *s daliServers.set(cfg->queryProp("Software/EspProcess/@daliServers")); const char *computer = cfg->queryProp("Software/EspProcess/@computer"); if (daliServers.isEmpty() || !computer || streq(computer, "localhost")) //otherwise can't assume environment "." netAddresses are the same as my address - queryHostIP().getIpText(envLocalAddress); + queryHostIP().getHostText(envLocalAddress); else { //a bit weird, but other netAddresses in the environment are not the same localhost as this server @@ -1478,7 +1478,7 @@ bool getWsWuInfoFromSasha(IEspContext &context, SocketEndpoint &ep, const char* StringBuffer url; throw MakeStringException(ECLWATCH_CANNOT_CONNECT_ARCHIVE_SERVER, "Sasha (%s) took too long to respond from: Get information for %s.", - ep.getUrlStr(url).str(), wuid); + ep.getEndpointHostText(url).str(), wuid); } if (cmd->numIds()==0) @@ -2482,7 +2482,7 @@ class CArchivedWUsReader : public CInterface, implements IArchivedWUsReader StringBuffer url; throw MakeStringException(ECLWATCH_CANNOT_CONNECT_ARCHIVE_SERVER, "Sasha (%s) took too long to respond from: Get archived workUnits.", - ep.getUrlStr(url).str()); + ep.getEndpointHostText(url).str()); } numberOfWUsReturned = cmd->numIds(); @@ -4914,7 +4914,7 @@ bool CWsWorkunitsEx::onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfo if (zapInfoReq.esp.isEmpty()) { IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(zapInfoReq.esp); + ipaddr.getHostText(zapInfoReq.esp); } zapInfoReq.thor = req.getThorIPAddress(); } @@ -4982,7 +4982,7 @@ bool CWsWorkunitsEx::onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoReques else { IpAddress ipaddr = queryHostIP(); - ipaddr.getIpText(EspIP); + ipaddr.getHostText(EspIP); resp.setESPIPAddress(EspIP.str()); } if ((version >= 1.96) && !queryRemoteLogAccessor()) diff --git a/esp/smc/SMCLib/TpContainer.cpp b/esp/smc/SMCLib/TpContainer.cpp index ee72b450357..8d504dbe9de 100644 --- a/esp/smc/SMCLib/TpContainer.cpp +++ b/esp/smc/SMCLib/TpContainer.cpp @@ -263,7 +263,7 @@ void CTpWrapper::appendThorMachineList(double clientVersion, IConstEnvironment* const char* machineType, unsigned& processNumber, unsigned channels, const char* directory, IArrayOf& machineList) { StringBuffer netAddress; - node.endpoint().getIpText(netAddress); + node.endpoint().getHostText(netAddress); if (netAddress.length() == 0) { OWARNLOG("Net address not found for a node of %s", clusterName); @@ -418,7 +418,7 @@ IEspTpMachine* CTpWrapper::createTpMachineEx(const char* name, const char* type, StringBuffer networkAddress; IpAddress ipAddr; ipAddr.ipset(netAddr.str()); - ipAddr.getIpText(networkAddress); + ipAddr.getHostText(networkAddress); machine->setNetaddress(networkAddress.str()); } diff --git a/esp/smc/SMCLib/TpWrapper.cpp b/esp/smc/SMCLib/TpWrapper.cpp index 9139c36e6f2..ee2baae2ab0 100644 --- a/esp/smc/SMCLib/TpWrapper.cpp +++ b/esp/smc/SMCLib/TpWrapper.cpp @@ -1349,7 +1349,7 @@ void CTpWrapper::appendThorMachineList(double clientVersion, IConstEnvironment* const char* machineType, unsigned& processNumber, unsigned channels, const char* directory, IArrayOf& machineList) { StringBuffer netAddress; - node.endpoint().getIpText(netAddress); + node.endpoint().getHostText(netAddress); if (netAddress.length() == 0) { OWARNLOG("Net address not found for a node of %s", clusterName); @@ -1734,7 +1734,7 @@ IEspTpMachine* CTpWrapper::createTpMachineEx(const char* name, const char* type, StringBuffer networkAddress; IpAddress ipAddr; ipAddr.ipset(netAddr.str()); - ipAddr.getIpText(networkAddress); + ipAddr.getHostText(networkAddress); machine->setNetaddress(networkAddress.str()); } diff --git a/esp/src/eclwatch/TargetSelectClass.js b/esp/src/eclwatch/TargetSelectClass.js index 6e33c0fecdd..362e35a8a7b 100644 --- a/esp/src/eclwatch/TargetSelectClass.js +++ b/esp/src/eclwatch/TargetSelectClass.js @@ -587,27 +587,25 @@ define([ loadLogs: function (params) { var context = this; this.set("options", []); - FileSpray.FileList({ + WsTopology.TpListLogFiles({ request: { - Mask: "*.log", - Netaddr: params.treeNode.newPreflight ? params.treeNode.NetAddress : params.treeNode.getNetaddress(), - OS: params.treeNode.newPreflight ? params.treeNode.OS : params.treeNode.getOS(), + NetworkAddress: params.treeNode.newPreflight ? params.treeNode.NetAddress : params.treeNode.getNetaddress(), Path: params.treeNode.newPreflight ? params.treeNode.LogDirectory : params.treeNode.getLogDirectory() } }).then(function (response) { - if (lang.exists("FileListResponse.files.PhysicalFileStruct", response)) { + if (lang.exists("TpListLogFilesResponse.Files.LogFileStruct", response)) { var options = []; - var targetData = response.FileListResponse.files.PhysicalFileStruct; + var targetData = response.TpListLogFilesResponse.Files.LogFileStruct; var shortestLabelLen = 9999; var shortestLabel = ""; for (var i = 0; i < targetData.length; ++i) { options.push({ - label: targetData[i].name, // + " " + targetData[i].filesize + " " + targetData[i].modifiedtime, - value: targetData[i].name + label: targetData[i].Name, // + " " + targetData[i].filesize + " " + targetData[i].modifiedtime, + value: targetData[i].Name }); - if (shortestLabelLen > targetData[i].name.length) { - shortestLabelLen = targetData[i].name.length; - shortestLabel = targetData[i].name; + if (shortestLabelLen > targetData[i].Name.length) { + shortestLabelLen = targetData[i].Name.length; + shortestLabel = targetData[i].Name; } } options.sort(function (l, r) { diff --git a/esp/src/package-lock.json b/esp/src/package-lock.json index a137710f749..94c6a6c9e94 100644 --- a/esp/src/package-lock.json +++ b/esp/src/package-lock.json @@ -14,21 +14,21 @@ "@fluentui/react-experiments": "8.14.95", "@fluentui/react-hooks": "8.6.29", "@fluentui/react-icons-mdl2": "1.3.47", - "@hpcc-js/chart": "2.81.4", - "@hpcc-js/codemirror": "2.60.9", - "@hpcc-js/common": "2.71.9", - "@hpcc-js/comms": "2.83.2", - "@hpcc-js/dataflow": "8.1.4", - "@hpcc-js/eclwatch": "2.73.22", - "@hpcc-js/graph": "2.85.5", - "@hpcc-js/html": "2.42.10", - "@hpcc-js/layout": "2.49.9", - "@hpcc-js/map": "2.77.8", - "@hpcc-js/other": "2.15.9", - "@hpcc-js/phosphor": "2.16.9", - "@hpcc-js/react": "2.53.6", - "@hpcc-js/tree": "2.40.9", - "@hpcc-js/util": "2.50.3", + "@hpcc-js/chart": "2.81.6", + "@hpcc-js/codemirror": "2.60.11", + "@hpcc-js/common": "2.71.11", + "@hpcc-js/comms": "2.84.3", + "@hpcc-js/dataflow": "8.1.6", + "@hpcc-js/eclwatch": "2.73.26", + "@hpcc-js/graph": "2.85.7", + "@hpcc-js/html": "2.42.12", + "@hpcc-js/layout": "2.49.11", + "@hpcc-js/map": "2.77.10", + "@hpcc-js/other": "2.15.11", + "@hpcc-js/phosphor": "2.16.11", + "@hpcc-js/react": "2.53.8", + "@hpcc-js/tree": "2.40.11", + "@hpcc-js/util": "2.50.5", "@kubernetes/client-node": "0.18.1", "clipboard": "2.0.11", "d3-dsv": "3.0.1", @@ -1520,37 +1520,37 @@ } }, "node_modules/@hpcc-js/api": { - "version": "2.12.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.9.tgz", - "integrity": "sha512-e5OY9fN8h+UGKrM5J3Z8+OFsXuYW6kH+PQcM0qcvFuIzuYp0UsDF1prKXPCDIutHvJK476EyXFZZayaw+/RGnQ==", + "version": "2.12.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/api/-/api-2.12.11.tgz", + "integrity": "sha512-Y2qisKnaMCwmqnKSGNF7yrzGcqP7ZMiNnWh2SrP1RkTkYskUcICEKgx0TUg9LgK58irfGGi4Y1bSiJgLNbC4eA==", "dependencies": { - "@hpcc-js/common": "^2.71.9" + "@hpcc-js/common": "^2.71.11" } }, "node_modules/@hpcc-js/chart": { - "version": "2.81.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.81.4.tgz", - "integrity": "sha512-uO6beO8aaDy47Vmx5kFJSnFI4EAMItiKniDCxZAG2uu61Vaqt3U+OIA9vfTIqOS/Z/MiuSnspgbmHR/SyRF+sQ==", + "version": "2.81.6", + "resolved": "https://registry.npmjs.org/@hpcc-js/chart/-/chart-2.81.6.tgz", + "integrity": "sha512-UpefYA3lp+oCeHjz7QiHlkSDeDdh0FfuikbSZAzcCDOuSNEiLfvyFlOaqM9Bdgq/rfiuXvVbiZRgu0CmkVtaQQ==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/codemirror": { - "version": "2.60.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.60.9.tgz", - "integrity": "sha512-36jMY/zrPevtfWsAwTwKAMrU+SqR0Ej/iXpwK4ixCWKu/EMf6kEZLl0esendEvK4r3znDWLXhqAC1hveNuD4cw==", + "version": "2.60.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/codemirror/-/codemirror-2.60.11.tgz", + "integrity": "sha512-uaPj3bWS/j8nLm9/naz572G3DrJCfA3jfk3m7iwuEeK0fy7tOhYFqkipjxFKVnE3vRzMTN9s8MA1sNDZVVOoYA==", "dependencies": { - "@hpcc-js/common": "^2.71.9" + "@hpcc-js/common": "^2.71.11" } }, "node_modules/@hpcc-js/common": { - "version": "2.71.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.9.tgz", - "integrity": "sha512-TM3wGyJKgWtYlC+waQpuwhxZaqDbcbtXiBCwYAHeA4704WJr0UGkvTMo9znkqR3mrWpn5yxZGCV+o5QBRcWUgw==", + "version": "2.71.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/common/-/common-2.71.11.tgz", + "integrity": "sha512-Z3gtybbLMNOgge4yiG4pUfkY29kHxA//H99dwanNeCvujzpXaJnmnoTpLdZnSj3C/cjQUxsgQGFO4xI0qvzvpQ==", "dependencies": { - "@hpcc-js/util": "^2.50.3", + "@hpcc-js/util": "^2.50.5", "@types/d3-array": "1.2.9", "@types/d3-brush": "1.1.5", "@types/d3-collection": "1.0.10", @@ -1569,17 +1569,18 @@ } }, "node_modules/@hpcc-js/comms": { - "version": "2.83.2", - "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.83.2.tgz", - "integrity": "sha512-STLvMEIXQUSrgNGbNkimlNf4Y1Yso52jnWyCRc/uZUN8cEd7z3Jim6Xj+cjKizrRfDVBRbg006H5HzaEbUu7Jg==", + "version": "2.84.3", + "resolved": "https://registry.npmjs.org/@hpcc-js/comms/-/comms-2.84.3.tgz", + "integrity": "sha512-PL1K9fdxN++UtX1UPMW4Jn5JO1IefxHX08lJcCZ7ekuHp3+brrokj2luBQ3Z8uQPXEEXKSdwYoP7pTyX5pyrtg==", "dependencies": { - "@hpcc-js/ddl-shim": "^2.20.3", - "@hpcc-js/util": "^2.50.3", + "@hpcc-js/ddl-shim": "^2.20.5", + "@hpcc-js/util": "^2.50.5", "@xmldom/xmldom": "0.8.8", "abort-controller": "3.0.0", - "node-fetch": "2.6.11", + "node-fetch": "2.7.0", "safe-buffer": "5.2.1", - "tmp": "0.2.1" + "tmp": "0.2.1", + "undici": "5.24.0" } }, "node_modules/@hpcc-js/comms/node_modules/safe-buffer": { @@ -1602,14 +1603,14 @@ ] }, "node_modules/@hpcc-js/dataflow": { - "version": "8.1.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/dataflow/-/dataflow-8.1.4.tgz", - "integrity": "sha512-T6rz2FqwI+suNwCzvVTHwtGBAtxm5L+aNnnXjJpXO5+KzvdtIAOV+1+WBuRCAkogj/gRdhEwW3XPzK5x3DbYkw==" + "version": "8.1.6", + "resolved": "https://registry.npmjs.org/@hpcc-js/dataflow/-/dataflow-8.1.6.tgz", + "integrity": "sha512-BMmpA6CP00nRDdeq7MS/LRN+e08dyBnnLWEITK5zuEn8D9sFo4ZJlmrKNp+Lcox/m+CuNqWMTG9Z5c+hK2jUyw==" }, "node_modules/@hpcc-js/ddl-shim": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/@hpcc-js/ddl-shim/-/ddl-shim-2.20.3.tgz", - "integrity": "sha512-2ZS1am7xCmLxzezM3CqTHmaPlAU7NQMQ4v6B5Mv7BMy6mkrSMeP2VAYYtNbwjrMUSa0ZFw4pimPFd3QWEH7bJQ==", + "version": "2.20.5", + "resolved": "https://registry.npmjs.org/@hpcc-js/ddl-shim/-/ddl-shim-2.20.5.tgz", + "integrity": "sha512-TgUzwlzRCS0Tl1Nc8FQ185mbAE11/ExRFAYtkYjqN0Lf45PwDoRoIoPQlqSnIis2Br7SNVmZPAO75poWpf0qLg==", "hasInstallScript": true, "dependencies": { "ajv": "6.12.6" @@ -1619,79 +1620,79 @@ } }, "node_modules/@hpcc-js/dgrid": { - "version": "2.32.10", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.10.tgz", - "integrity": "sha512-/4ETFmBBHcVtOWGvrEV2vgWf8+YZOFPH/nKZofRJCGCWvvJrdmhjFGV3PDXHO91uOrObLnHI1dppBJ1B+E2j6A==", + "version": "2.32.12", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid/-/dgrid-2.32.12.tgz", + "integrity": "sha512-A6S1YD2bkMmGLQ/gFJG0hBXn9fMXUE10SejQq/7In3NpHm3vOBwD2Pv0Ak7/md6TRjmH1nN8nT0PjK2wERY4Pg==", "dependencies": { - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/ddl-shim": "^2.20.3", - "@hpcc-js/dgrid-shim": "^2.24.4", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/ddl-shim": "^2.20.5", + "@hpcc-js/dgrid-shim": "^2.24.6", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/dgrid-shim": { - "version": "2.24.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.24.4.tgz", - "integrity": "sha512-R0eEJaG7HRD/Rdu8BOjCBFCn69rQuawsGS3h4pCs5ns+aVtvjb1/yyUid2UWNrE+Lf/x9ZyvC/kqHzvZ/zpuZQ==" + "version": "2.24.6", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid-shim/-/dgrid-shim-2.24.6.tgz", + "integrity": "sha512-WiY1nH6QJPftxEEpWXQIn5uGxcx/8MAgfLbGOQt374DlSpIkyBLskX0iPw8+lS5UGLEh53PiirPkdRkCLRTCBQ==" }, "node_modules/@hpcc-js/dgrid2": { - "version": "2.3.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.9.tgz", - "integrity": "sha512-CHzc5okZY163Si9um7FZb+p1XmkFqr3itUEfqD8MvZtZc4uK/DxUl2YlXiUNwYeMvj26aN9ZW94k22oNLPVQ/Q==", + "version": "2.3.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/dgrid2/-/dgrid2-2.3.11.tgz", + "integrity": "sha512-UvNambdiVy8OQ2U4nNl2QKjUjEKI2FVFc5QX4qD5XaTXCPUPEGYH0CHlNoGfOKL/hDQxUgsxLxsF9iBuVYP7IQ==", "dependencies": { - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/preact-shim": "^2.16.4", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/preact-shim": "^2.16.6", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/eclwatch": { - "version": "2.73.22", - "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.73.22.tgz", - "integrity": "sha512-2CoNKaB79heMRv+/uNuza11YNP49vc8GQF+fjE1XCCj4iUI1q3aBPh/uE2C7u6xZ1brgTWfm4h2ocfTsLyUpUA==", - "dependencies": { - "@hpcc-js/codemirror": "^2.60.9", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/comms": "^2.83.2", - "@hpcc-js/dgrid": "^2.32.10", - "@hpcc-js/graph": "^2.85.5", - "@hpcc-js/layout": "^2.49.9", - "@hpcc-js/phosphor": "^2.16.9", - "@hpcc-js/timeline": "^2.51.12", - "@hpcc-js/tree": "^2.40.9", - "@hpcc-js/util": "^2.50.3" + "version": "2.73.26", + "resolved": "https://registry.npmjs.org/@hpcc-js/eclwatch/-/eclwatch-2.73.26.tgz", + "integrity": "sha512-h2bW8tb+byebxotQetk6wiJ0Aj6tuHpQrHp/BUk7mvQbR0lJEPXSZ5cIQgIaysDXzdUrfMA6sh10crbR/9t5Kw==", + "dependencies": { + "@hpcc-js/codemirror": "^2.60.11", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/comms": "^2.84.3", + "@hpcc-js/dgrid": "^2.32.12", + "@hpcc-js/graph": "^2.85.7", + "@hpcc-js/layout": "^2.49.11", + "@hpcc-js/phosphor": "^2.16.11", + "@hpcc-js/timeline": "^2.51.14", + "@hpcc-js/tree": "^2.40.11", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/graph": { - "version": "2.85.5", - "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.5.tgz", - "integrity": "sha512-gzyKOlaJteM2/WskfWKNFF+qQU4q9pLOgQb2C4aZ+iMe22ErIlscuk4++80JFD2QUmSCG8t3oykL+gtcWz0exQ==", + "version": "2.85.7", + "resolved": "https://registry.npmjs.org/@hpcc-js/graph/-/graph-2.85.7.tgz", + "integrity": "sha512-osewnAm3nauQQUK0rbXAZ2hMmqGAmucfZzNIuiqqvx10DyU+XC+t1aRWDrUrZ8gRRFW0ObYetI8fv2/35twadw==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/html": "^2.42.10", - "@hpcc-js/react": "^2.53.6", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/html": "^2.42.12", + "@hpcc-js/react": "^2.53.8", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/html": { - "version": "2.42.10", - "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.10.tgz", - "integrity": "sha512-oz9I2QZV8U7DUPuWe7oItgJVnc054ygE/NaI4knMVedhJT0Eo758/Vgg8t4g3EsEVzji26sSZOtc06ODuO87rw==", + "version": "2.42.12", + "resolved": "https://registry.npmjs.org/@hpcc-js/html/-/html-2.42.12.tgz", + "integrity": "sha512-Myu3CI7c/ESazZ70ZJTMxtpwJAOHgTmTr7h94uXM4J9uKhGxMVDIIDM7frvOskhDoD4vy+Mc+BrPCnfJsXdd7Q==", "dependencies": { - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/preact-shim": "^2.16.4", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/preact-shim": "^2.16.6", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/layout": { - "version": "2.49.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.9.tgz", - "integrity": "sha512-2DOitZ8KbgOVKC46NU3b9PQ/zYPLoC+ekUnYOEUv4ADBgQmqq+HGmYXSiDx23wLROrvUX4JNC7ciN5vzYd+02g==", + "version": "2.49.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/layout/-/layout-2.49.11.tgz", + "integrity": "sha512-L+UzIaRQdI4gj5zVpyOjvD8V4TGZ7Afr5GccpNbazUyjwyIDYBEObwFkDYHOd3anfAx/xFzf/S3vgoiXe5JfoQ==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/chart": "^2.81.4", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/dgrid2": "^2.3.9" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/chart": "^2.81.6", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/dgrid2": "^2.3.11" } }, "node_modules/@hpcc-js/leaflet-shim": { @@ -1704,44 +1705,44 @@ } }, "node_modules/@hpcc-js/map": { - "version": "2.77.8", - "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.8.tgz", - "integrity": "sha512-M0AljPl4sZswR3U8qUDiHLVG537YZPuT5Z/uecFpgiWW9gHZy2hu4QNRqm+o6DfsOPcIsqXNy5UVnr+EvylFxQ==", - "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/graph": "^2.85.5", - "@hpcc-js/layout": "^2.49.9", + "version": "2.77.10", + "resolved": "https://registry.npmjs.org/@hpcc-js/map/-/map-2.77.10.tgz", + "integrity": "sha512-pghIGSMAJwy01xpfN9geTUUy0E6/KJeUUOt7YVn4D0D1OkHQJLr9nxwSz1tID76F0psUnRKiY7mfDUvhhZUbSA==", + "dependencies": { + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/graph": "^2.85.7", + "@hpcc-js/layout": "^2.49.11", "@hpcc-js/leaflet-shim": "^2.3.3", - "@hpcc-js/other": "^2.15.9", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/other": "^2.15.11", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/other": { - "version": "2.15.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.9.tgz", - "integrity": "sha512-KAOtYVGQqOC1OyLzDDmRlwle+Xfutel4LvmedMeosHZngvTgDN7biscp5viPJQxy2IF0odnuIlqu0MG+vZEpCg==", + "version": "2.15.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/other/-/other-2.15.11.tgz", + "integrity": "sha512-kRuHGmilmhPtzndoe4JFDOjeY7gQv2qDGFEXHoctv9arTvUESggO0C5zH2QO3FvJRhwu1SbXN15q6seeTkaP+A==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/layout": "^2.49.9" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/layout": "^2.49.11" } }, "node_modules/@hpcc-js/phosphor": { - "version": "2.16.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.16.9.tgz", - "integrity": "sha512-qx8kBjqw4llcygTMpzn++jbwzNofUibku8EUy6LBMSLE6Z5jQvghXAJqp/onmcyg0+4uwD8t/kKvKEDlstIiOQ==", + "version": "2.16.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor/-/phosphor-2.16.11.tgz", + "integrity": "sha512-AZ3IuBNgt9zoBUUPGR/5l/QBQT6Dbs7LzwB2UfIKri7xP+eYHZpy8b0h4GYBmuUaf6xp3usA2txcxdKvHhkySw==", "dependencies": { - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/other": "^2.15.9", - "@hpcc-js/phosphor-shim": "^2.14.4", - "@hpcc-js/util": "^2.50.3" + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/other": "^2.15.11", + "@hpcc-js/phosphor-shim": "^2.14.6", + "@hpcc-js/util": "^2.50.5" } }, "node_modules/@hpcc-js/phosphor-shim": { - "version": "2.14.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.14.4.tgz", - "integrity": "sha512-6kuLkLlyAUEeeu2Jgc1SSaou5D9Ks3tTVOLTnktzlZvKMgq7L0K6jpsn9ts728ZCQ+P0VPKdzDY58yVVdBt98g==", + "version": "2.14.6", + "resolved": "https://registry.npmjs.org/@hpcc-js/phosphor-shim/-/phosphor-shim-2.14.6.tgz", + "integrity": "sha512-PUCt29OdcMY/48kbBVvHQyZ+Dhr2MTvWlpQI2K04CIovZZ6Z7lkEmzRxf+VGfM9vhGWP9Amb6tf3MFrZK2+WSw==", "dependencies": { "@lumino/algorithm": "1.9.2", "@lumino/commands": "1.21.1", @@ -1750,48 +1751,48 @@ } }, "node_modules/@hpcc-js/preact-shim": { - "version": "2.16.4", - "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.16.4.tgz", - "integrity": "sha512-fuUXMWcOjCx3imFypR4CD9WLZSSU0MudbpIx5Zsb2ktQJagEOHOLuFnS57Ew9aTY9wLgwe1LM7JwLlr7MmPKaw==", + "version": "2.16.6", + "resolved": "https://registry.npmjs.org/@hpcc-js/preact-shim/-/preact-shim-2.16.6.tgz", + "integrity": "sha512-lpWJZ4CMSIA08rnhcUMG9ggktZHO89sQPFOvdZMMzwijIkXTTi8snn/JMkiGuzPgQvYuM6K2WTuve79zQl4wQA==", "dependencies": { "preact": "10.15.1" } }, "node_modules/@hpcc-js/react": { - "version": "2.53.6", - "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.6.tgz", - "integrity": "sha512-VEF7GsqzcEEC2EhuGmv87ORtE3PZpXef0cfppFmsMdRCEvPM4o4xpd025cVlVI7PWIG/vMafjJC8DR9iN7WjZA==", + "version": "2.53.8", + "resolved": "https://registry.npmjs.org/@hpcc-js/react/-/react-2.53.8.tgz", + "integrity": "sha512-NuuHR5R/Cf0GvKLDSb7/WKXWKeiu/1wsPIqRxuJKERq4YmZFuK1DNYkWdJskfOFlTIoRzNG5+EwyUqO1lvUPZg==", "dependencies": { - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/preact-shim": "^2.16.4" + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/preact-shim": "^2.16.6" } }, "node_modules/@hpcc-js/timeline": { - "version": "2.51.12", - "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.12.tgz", - "integrity": "sha512-GSNaDgex61pCB8MggI0QakJ12lLImsm4H8GKbbfV6vcYr8/vSkDQ+f6RXm6gWW6kV0mrCOQlntA9gh2gFSWNGQ==", + "version": "2.51.14", + "resolved": "https://registry.npmjs.org/@hpcc-js/timeline/-/timeline-2.51.14.tgz", + "integrity": "sha512-uamRb2RABNfNRXNvSDqqp3LtlwJqVS4w/gZY7rrs5jbG28ndFVWHCWEGHZDhNdmDAgG7fDAPQ07UqZxmeKBSWQ==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/chart": "^2.81.4", - "@hpcc-js/common": "^2.71.9", - "@hpcc-js/html": "^2.42.10", - "@hpcc-js/layout": "^2.49.9", - "@hpcc-js/react": "^2.53.6" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/chart": "^2.81.6", + "@hpcc-js/common": "^2.71.11", + "@hpcc-js/html": "^2.42.12", + "@hpcc-js/layout": "^2.49.11", + "@hpcc-js/react": "^2.53.8" } }, "node_modules/@hpcc-js/tree": { - "version": "2.40.9", - "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.9.tgz", - "integrity": "sha512-zyHgQhk9bZPlm65UdKz6DRfjLnGLur6wraOyN6DjsoiI342MVSJVC4MosolPTEq7xDpuJ1SEU49+qzlWqAXcKQ==", + "version": "2.40.11", + "resolved": "https://registry.npmjs.org/@hpcc-js/tree/-/tree-2.40.11.tgz", + "integrity": "sha512-obcT5PZpIe7rJYryIdsZb19ZfApsTRZmD5A08QkBcbyQ8zl1T2dXDj9JZ6rVCZcoBFjjegdS4Vr7EN9eu93XJw==", "dependencies": { - "@hpcc-js/api": "^2.12.9", - "@hpcc-js/common": "^2.71.9" + "@hpcc-js/api": "^2.12.11", + "@hpcc-js/common": "^2.71.11" } }, "node_modules/@hpcc-js/util": { - "version": "2.50.3", - "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.50.3.tgz", - "integrity": "sha512-zgwLaBnpXBJRd4RUYV/MHe2rP329tz4NR8/g5T7W6/VXlPBXR6fbZzy35Zsf7xv4yiTtTN27hi/7cgU5J3HOjA==", + "version": "2.50.5", + "resolved": "https://registry.npmjs.org/@hpcc-js/util/-/util-2.50.5.tgz", + "integrity": "sha512-TbEhzAuirvyhP2eleFuPuh4mgmK9YG1gLtAQJudr2bN0Q+3hZpepIkEq3kI8UHZ7F+nIBrVWbCYmA6/BrYKCbw==", "dependencies": { "tslib": "2.4.1" } @@ -3011,6 +3012,17 @@ "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "dev": true }, + "node_modules/busboy": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", + "integrity": "sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==", + "dependencies": { + "streamsearch": "^1.1.0" + }, + "engines": { + "node": ">=10.16.0" + } + }, "node_modules/byline": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/byline/-/byline-5.0.0.tgz", @@ -6384,9 +6396,9 @@ "dev": true }, "node_modules/node-fetch": { - "version": "2.6.11", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.11.tgz", - "integrity": "sha512-4I6pdBY1EthSqDmJkiNk3JIT8cswwR9nfeW/cPdUagJYEQG7R95WRH74wpz7ma8Gh/9dI9FP+OU+0E4FvtA55w==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -8079,6 +8091,14 @@ "safe-buffer": "~5.1.0" } }, + "node_modules/streamsearch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", + "integrity": "sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/strict-uri-encode": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz", @@ -8638,6 +8658,17 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" }, + "node_modules/undici": { + "version": "5.24.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.24.0.tgz", + "integrity": "sha512-OKlckxBjFl0oXxcj9FU6oB8fDAaiRUq+D8jrFWGmOfI/gIyjk/IeS75LMzgYKUaeHzLUcYvf9bbJGSrUwTfwwQ==", + "dependencies": { + "busboy": "^1.6.0" + }, + "engines": { + "node": ">=14.0" + } + }, "node_modules/universal-router": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/universal-router/-/universal-router-9.1.0.tgz", diff --git a/esp/src/package.json b/esp/src/package.json index 29a6c910bbd..90ad2068a7a 100644 --- a/esp/src/package.json +++ b/esp/src/package.json @@ -39,21 +39,21 @@ "@fluentui/react-experiments": "8.14.95", "@fluentui/react-hooks": "8.6.29", "@fluentui/react-icons-mdl2": "1.3.47", - "@hpcc-js/chart": "2.81.4", - "@hpcc-js/codemirror": "2.60.9", - "@hpcc-js/common": "2.71.9", - "@hpcc-js/comms": "2.83.2", - "@hpcc-js/dataflow": "8.1.4", - "@hpcc-js/eclwatch": "2.73.22", - "@hpcc-js/graph": "2.85.5", - "@hpcc-js/html": "2.42.10", - "@hpcc-js/layout": "2.49.9", - "@hpcc-js/map": "2.77.8", - "@hpcc-js/other": "2.15.9", - "@hpcc-js/phosphor": "2.16.9", - "@hpcc-js/react": "2.53.6", - "@hpcc-js/tree": "2.40.9", - "@hpcc-js/util": "2.50.3", + "@hpcc-js/chart": "2.81.6", + "@hpcc-js/codemirror": "2.60.11", + "@hpcc-js/common": "2.71.11", + "@hpcc-js/comms": "2.84.3", + "@hpcc-js/dataflow": "8.1.6", + "@hpcc-js/eclwatch": "2.73.26", + "@hpcc-js/graph": "2.85.7", + "@hpcc-js/html": "2.42.12", + "@hpcc-js/layout": "2.49.11", + "@hpcc-js/map": "2.77.10", + "@hpcc-js/other": "2.15.11", + "@hpcc-js/phosphor": "2.16.11", + "@hpcc-js/react": "2.53.8", + "@hpcc-js/tree": "2.40.11", + "@hpcc-js/util": "2.50.5", "@kubernetes/client-node": "0.18.1", "clipboard": "2.0.11", "d3-dsv": "3.0.1", diff --git a/esp/src/src-react/components/Files.tsx b/esp/src/src-react/components/Files.tsx index 03f60dff097..b2664d6b119 100644 --- a/esp/src/src-react/components/Files.tsx +++ b/esp/src/src-react/components/Files.tsx @@ -31,7 +31,7 @@ const FilterFields: Fields = { "LogicalFiles": { type: "checkbox", label: nlsHPCC.LogicalFiles }, "SuperFiles": { type: "checkbox", label: nlsHPCC.SuperFiles }, "Indexes": { type: "checkbox", label: nlsHPCC.Indexes }, - "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles }, + "NotInSuperfiles": { type: "checkbox", label: nlsHPCC.NotInSuperfiles, disabled: (params: Fields) => !!params?.SuperFiles?.value || !!params?.LogicalFiles?.value }, "NodeGroup": { type: "target-group", label: nlsHPCC.Cluster, placeholder: nlsHPCC.Cluster }, "FileSizeFrom": { type: "string", label: nlsHPCC.FromSizes, placeholder: "4096" }, "FileSizeTo": { type: "string", label: nlsHPCC.ToSizes, placeholder: "16777216" }, diff --git a/esp/src/src-react/components/Workunits.tsx b/esp/src/src-react/components/Workunits.tsx index 0f1e9314ed6..f2bfc4d4d84 100644 --- a/esp/src/src-react/components/Workunits.tsx +++ b/esp/src/src-react/components/Workunits.tsx @@ -150,25 +150,25 @@ export const Workunits: React.FunctionComponent = ({ label: nlsHPCC.TotalClusterTime, width: 120, justify: "right", }, - CompileCost: { + "Compile Cost": { label: nlsHPCC.CompileCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.CompileCost)}`; } }, - ExecuteCost: { + "Execute Cost": { label: nlsHPCC.ExecuteCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.ExecuteCost)}`; } }, - FileAccessCost: { + "File Access Cost": { label: nlsHPCC.FileAccessCost, width: 100, justify: "right", formatter: (cost, row) => { - return `${formatCost(cost)}`; + return `${formatCost(row.FileAccessCost)}`; } } }; diff --git a/esp/src/src-react/components/forms/Fields.tsx b/esp/src/src-react/components/forms/Fields.tsx index 3d191882b36..9f394b0aacc 100644 --- a/esp/src/src-react/components/forms/Fields.tsx +++ b/esp/src/src-react/components/forms/Fields.tsx @@ -490,12 +490,18 @@ export const TargetGroupTextField: React.FunctionComponent { TpGroupQuery({}).then(({ TpGroupQueryResponse }) => { - setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(n => { - return { - key: n.Name, - text: n.Name + (n.Name !== n.Kind ? ` (${n.Kind})` : "") - }; - })); + setTargetGroups(TpGroupQueryResponse.TpGroups.TpGroup.map(group => { + switch (group?.Kind) { + case "Thor": + case "hthor": + case "Roxie": + case "Plane": + return { + key: group.Name, + text: group.Name + (group.Name !== group.Kind ? ` (${group.Kind})` : "") + }; + } + }).filter(group => group)); }).catch(err => logger.error(err)); }, []); @@ -780,6 +786,7 @@ export function createInputs(fields: Fields, onChange?: (id: string, newValue: a label: field.label, field: onChange(fieldID, newValue)} /> diff --git a/esp/src/src-react/hooks/platform.ts b/esp/src/src-react/hooks/platform.ts index a987410b3ec..9a278eaff34 100644 --- a/esp/src/src-react/hooks/platform.ts +++ b/esp/src/src-react/hooks/platform.ts @@ -1,6 +1,6 @@ import * as React from "react"; import { scopedLogger } from "@hpcc-js/util"; -import { Topology, TpLogicalClusterQuery } from "@hpcc-js/comms"; +import { Topology, WsTopology } from "@hpcc-js/comms"; import { getBuildInfo, BuildInfo } from "src/Session"; import { cmake_build_type, containerized } from "src/BuildInfo"; @@ -33,9 +33,9 @@ export function useBuildInfo(): [BuildInfo, { isContainer: boolean, currencyCode return [buildInfo, { isContainer, currencyCode, opsCategory }]; } -export function useLogicalClusters(): [TpLogicalClusterQuery.TpLogicalCluster[] | undefined, TpLogicalClusterQuery.TpLogicalCluster | undefined] { - const [targetClusters, setTargetClusters] = React.useState(); - const [defaultCluster, setDefaultCluster] = React.useState(); +export function useLogicalClusters(): [WsTopology.TpLogicalCluster[] | undefined, WsTopology.TpLogicalCluster | undefined] { + const [targetClusters, setTargetClusters] = React.useState(); + const [defaultCluster, setDefaultCluster] = React.useState(); React.useEffect(() => { const topology = Topology.attach({ baseUrl: "" }); @@ -43,9 +43,9 @@ export function useLogicalClusters(): [TpLogicalClusterQuery.TpLogicalCluster[] topology.fetchLogicalClusters().then(response => { if (active) { setTargetClusters(response); - let firstRow: TpLogicalClusterQuery.TpLogicalCluster; - let firstHThor: TpLogicalClusterQuery.TpLogicalCluster; - let firstThor: TpLogicalClusterQuery.TpLogicalCluster; + let firstRow: WsTopology.TpLogicalCluster; + let firstHThor: WsTopology.TpLogicalCluster; + let firstThor: WsTopology.TpLogicalCluster; response.forEach(row => { if (firstRow === undefined) { firstRow = row; diff --git a/esp/src/src/WsTopology.ts b/esp/src/src/WsTopology.ts index d23e4bc97f3..d70e6f954af 100644 --- a/esp/src/src/WsTopology.ts +++ b/esp/src/src/WsTopology.ts @@ -233,6 +233,9 @@ export function TpGetComponentFile(params) { params.handleAs = "text"; return ESPRequest.send("WsTopology", "TpGetComponentFile", params); } +export function TpListLogFiles(params) { + return ESPRequest.send("WsTopology", "TpListLogFiles", params); +} export function TpLogFile(params) { return ESPRequest.send("WsTopology", "TpLogFile", params); } diff --git a/esp/src/src/store/Memory.ts b/esp/src/src/store/Memory.ts index ec36b30414c..1bec0fee3e6 100644 --- a/esp/src/src/store/Memory.ts +++ b/esp/src/src/store/Memory.ts @@ -69,7 +69,7 @@ export class Memory extends BaseStore { } } - protected fetchData(request: QueryRequest, options: QueryOptions): ThenableResponse { + protected fetchData(request: QueryRequest, options: QueryOptions = {}): ThenableResponse { options.alphanumColumns = this.alphanumSort; const data = this.queryEngine(request, options)(this.data); data.total = this.data.length; diff --git a/esp/src/src/store/Store.ts b/esp/src/src/store/Store.ts index 41dea790b39..d41d24e893c 100644 --- a/esp/src/src/store/Store.ts +++ b/esp/src/src/store/Store.ts @@ -21,7 +21,7 @@ export interface QueryOptions { start?: number; count?: number; sort?: QuerySort; - alphanumColumns: { [id: string]: boolean }; + alphanumColumns?: { [id: string]: boolean }; } export abstract class BaseStore { diff --git a/esp/test/httptest/httptest.cpp b/esp/test/httptest/httptest.cpp index 343489b87d5..0888478912e 100644 --- a/esp/test/httptest/httptest.cpp +++ b/esp/test/httptest/httptest.cpp @@ -539,7 +539,7 @@ int HttpClient::sendRequest(int times, HttpStat& stat, StringBuffer& req) if(socket.get() == NULL) { StringBuffer urlstr; - OERRLOG("Can't connect to %s", ep.getUrlStr(urlstr).str()); + OERRLOG("Can't connect to %s", ep.getEndpointHostText(urlstr).str()); continue; } @@ -857,7 +857,7 @@ int COneServerHttpProxyThread::start() if(socket2.get() == NULL) { StringBuffer urlstr; - OERRLOG("Can't connect to %s", ep.getUrlStr(urlstr).str()); + OERRLOG("Can't connect to %s", ep.getEndpointHostText(urlstr).str()); return -1; } diff --git a/esp/tools/soapplus/http.cpp b/esp/tools/soapplus/http.cpp index b6280f51bdd..fd288668ef4 100644 --- a/esp/tools/soapplus/http.cpp +++ b/esp/tools/soapplus/http.cpp @@ -1789,7 +1789,7 @@ int HttpClient::sendRequest(StringBuffer& req, IFileIO* request_output, IFileIO* if(socket.get() == NULL) { StringBuffer urlstr; - OERRLOG("Can't connect to %s", ep.getUrlStr(urlstr).str()); + OERRLOG("Can't connect to %s", ep.getEndpointHostText(urlstr).str()); return -1; } diff --git a/esp/tools/soapplus/http.hpp b/esp/tools/soapplus/http.hpp index 8fb59ff5a37..060e107c0f5 100644 --- a/esp/tools/soapplus/http.hpp +++ b/esp/tools/soapplus/http.hpp @@ -122,7 +122,7 @@ class CAddress : public CInterface, implements IInterface m_addr->sin_port = htons(port); IpAddress ip(host); - ip.getIpText(m_ip); + ip.getHostText(m_ip); m_port = port; m_fqdn.set(host); diff --git a/esp/tools/soapplus/httpproxy.cpp b/esp/tools/soapplus/httpproxy.cpp index 102946455ea..b07874d8c7f 100644 --- a/esp/tools/soapplus/httpproxy.cpp +++ b/esp/tools/soapplus/httpproxy.cpp @@ -212,7 +212,7 @@ int COneServerHttpProxyThread::start() if(socket2.get() == NULL) { StringBuffer urlstr; - OERRLOG("Can't connect to %s", ep.getUrlStr(urlstr).str()); + OERRLOG("Can't connect to %s", ep.getEndpointHostText(urlstr).str()); return -1; } @@ -676,7 +676,7 @@ class CSocksProxyThread : public Thread ip.setNetAddress(4,inbuf+4); StringBuffer ipstr; - ip.getIpText(ipstr); + ip.getHostText(ipstr); char inbuf2[16]; m_client->read(inbuf2, 0, 16, lenread); diff --git a/fs/dafilesrv/dafilesrv.cpp b/fs/dafilesrv/dafilesrv.cpp index efbf863e5a6..0b1bbc3e5a3 100644 --- a/fs/dafilesrv/dafilesrv.cpp +++ b/fs/dafilesrv/dafilesrv.cpp @@ -395,7 +395,7 @@ int main(int argc, const char* argv[]) // Use the "public" certificate issuer, unless it's visibility is "cluster" (meaning internal only) const char *visibility = getComponentConfigSP()->queryProp("service/@visibility"); const char *certScope = strsame("cluster", visibility) ? "local" : "public"; - IPropertyTree *info = queryTlsSecretInfo(certScope); + Owned info = getIssuerTlsServerConfig(certScope); connectMethod = info ? SSLOnly : SSLNone; // NB: connectMethod will direct the CRemoteFileServer on accept to create a secure socket based on the same issuer certificates @@ -780,7 +780,7 @@ int main(int argc, const char* argv[]) if (listenep.isNull()) eps.append(listenep.port); else - listenep.getUrlStr(eps); + listenep.getEndpointHostText(eps); if (connectMethod != SSLOnly) PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on %s", eps.str()); @@ -792,7 +792,7 @@ int main(int argc, const char* argv[]) if (sslep.isNull()) eps.append(sslep.port); else - sslep.getUrlStr(eps); + sslep.getEndpointHostText(eps); PROGLOG("Opening " DAFS_SERVICE_DISPLAY_NAME " on SECURE %s", eps.str()); } @@ -849,7 +849,7 @@ int main(int argc, const char* argv[]) if (listenep.isNull()) eps.append(listenep.port); else - listenep.getUrlStr(eps); + listenep.getEndpointHostText(eps); if (connectMethod != SSLOnly) PROGLOG("Opening Dali File Server on %s", eps.str()); if (connectMethod == SSLOnly || connectMethod == SSLFirst || connectMethod == UnsecureFirst) @@ -860,7 +860,7 @@ int main(int argc, const char* argv[]) if (sslep.isNull()) eps.append(sslep.port); else - sslep.getUrlStr(eps); + sslep.getEndpointHostText(eps); PROGLOG("Opening Dali File Server on SECURE %s", eps.str()); } diff --git a/fs/dafsclient/rmtclient.cpp b/fs/dafsclient/rmtclient.cpp index 1a0b2123c55..7f69a7f0131 100644 --- a/fs/dafsclient/rmtclient.cpp +++ b/fs/dafsclient/rmtclient.cpp @@ -156,7 +156,7 @@ static ISecureSocket *createSecureSocket(ISocket *sock, const char *issuer) auto it = secureCtxClientIssuerMap.find(issuer); if (it == secureCtxClientIssuerMap.end()) { - IPropertyTree *info = queryTlsSecretInfo(issuer); + Owned info = getIssuerTlsServerConfig(issuer); if (!info) throw makeStringExceptionV(-1, "createSecureSocket() : missing MTLS configuration for issuer: %s", issuer); secureContext.setown(createSecureSocketContextEx2(info, ClientSocket)); @@ -683,7 +683,7 @@ void CRemoteBase::connectSocket(SocketEndpoint &ep, unsigned connectTimeoutMs, u if (msTick()-lastfailtimeRelease(); StringBuffer msg("Failed to connect (setting host down) to dafilesrv/daliservix on "); - ep.getUrlStr(msg); + ep.getEndpointHostText(msg); throw createDafsException(DAFSERR_connection_failed,msg.str()); } throw; @@ -923,7 +923,7 @@ void CRemoteBase::sendRemoteCommand(MemoryBuffer & src, MemoryBuffer & reply, bo StringBuffer msg; if (filename.get()) msg.append(filename); - ep.getUrlStr(msg.append('[')).append("] "); + ep.getEndpointHostText(msg.append('[')).append("] "); size32_t pos = reply.getPos(); if (pos bool reopen() { StringBuffer s; - PROGLOG("Attempting reopen of %s on %s",parent->queryLocalName(),parent->queryEp().getUrlStr(s).str()); + PROGLOG("Attempting reopen of %s on %s",parent->queryLocalName(),parent->queryEp().getEndpointHostText(s).str()); if (open(mode,compatmode,extraFlags)) return true; return false; @@ -1401,7 +1401,7 @@ class CRemoteFileIO : public CInterfaceOf if (errCode) { StringBuffer msg; - parent->ep.getUrlStr(msg.append('[')).append("] "); + parent->ep.getEndpointHostText(msg.append('[')).append("] "); if (replyBuffer.getPos()read(0,sizeof(buf)-1,buf); if ((rd!=sz)||(memcmp(buf,ds.str(),sz)!=0)) { StringBuffer s; - ep.getIpText(s); + ep.getHostText(s); throw MakeStringException(-1,"Data discrepancy on disk read of %s of %s",path.str(),s.str()); } } diff --git a/fs/dafscontrol/dafscontrol.cpp b/fs/dafscontrol/dafscontrol.cpp index 59b63f512fb..c1b701c5a56 100644 --- a/fs/dafscontrol/dafscontrol.cpp +++ b/fs/dafscontrol/dafscontrol.cpp @@ -142,7 +142,7 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali ISocket *sock = sockets.item(i); StringBuffer epstr; SocketEndpoint ep = eps.item(i); - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); // PROGLOG("T.1 %s %x",epstr.str(),(unsigned)sock); StringBuffer verstr; unsigned rver=0; @@ -216,7 +216,7 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali if (!quiet) { StringBuffer epstr; ForEachItemIn(i,result) { - result.item(i).getUrlStr(epstr.clear()); + result.item(i).getEndpointHostText(epstr.clear()); StringAttrItem &attr = resultstr.item(i); if (attr.text.length()==0) UERRLOG("%s: %s not running DAFILESRV",grpip,epstr.str()); @@ -239,7 +239,7 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali StringBuffer epstr; unsigned failed=0; ForEachItemIn(i,result) { - result.item(i).getUrlStr(epstr.clear()); + result.item(i).getEndpointHostText(epstr.clear()); StringAttrItem &attr = resultstr.item(i); if (attr.text.length()!=0) PROGLOG("%s,%s,%s",grpip,epstr.str(),attr.text.get()); @@ -254,7 +254,7 @@ unsigned applyNodes(const char *grpip, ApplyMode mode, unsigned ver, bool isdali if (!quiet) { StringBuffer epstr; ForEachItemIn(i,result) { - result.item(i).getUrlStr(epstr.clear()); + result.item(i).getEndpointHostText(epstr.clear()); UERRLOG("%s: %s not running DAFILESRV",grpip,epstr.str()); } unsigned numok = eps.ordinality()-result.ordinality(); @@ -375,7 +375,7 @@ int main(int argc, char* argv[]) if (ret!=0) UERRLOG("setDafileSvrTraceFlags returned %d",ret); StringBuffer s("done "); - ep.getUrlStr(s); + ep.getEndpointHostText(s); PROGLOG("%s",s.str()); } } @@ -395,7 +395,7 @@ int main(int argc, char* argv[]) if (!isdali||!getCluster(argv[ai+1],eps)) { SocketEndpoint ep(argv[ai+1]); StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); VStringBuffer result("Info for %s", epStr.str()); int ret = getDafileSvrInfo(ep, level, result); if (ret!=0) @@ -407,7 +407,7 @@ int main(int argc, char* argv[]) ForEachItemIn(ni,eps) { SocketEndpoint ep = eps.item(ni); StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); VStringBuffer result("Info for %s: ", epStr.str()); int ret = getDafileSvrInfo(ep, level, result); if (ret!=0) @@ -438,7 +438,7 @@ int main(int argc, char* argv[]) if (ret!=0) UERRLOG("setDafileSvrThrottleLimit returned %d, error = %s", ret, errMsg.str()); StringBuffer s("done "); - ep.getUrlStr(s); + ep.getEndpointHostText(s); PROGLOG("%s",s.str()); } } diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 3ce29430a25..c53ca376908 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -133,7 +133,7 @@ static ISecureSocket *createSecureSocket(ISocket *sock, bool disableClientCertVe */ const char *certScope = strsame("cluster", getComponentConfigSP()->queryProp("service/@visibility")) ? "local" : "public"; - IPropertyTree *info = queryTlsSecretInfo(certScope); + Owned info = getIssuerTlsServerConfig(certScope); if (!info) throw makeStringException(-1, "createSecureSocket() : missing MTLS configuration"); Owned cloneInfo; @@ -5367,7 +5367,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface else { StringBuffer ips; - listenep.getIpText(ips); + listenep.getHostText(ips); acceptSock.setown(ISocket::create_ip(listenep.port,ips.str())); } } @@ -5404,7 +5404,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface else { StringBuffer ips; - sslep.getIpText(ips); + sslep.getHostText(ips); secureSock.setown(ISocket::create_ip(sslep.port,ips.str())); } } @@ -5419,7 +5419,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface else { StringBuffer ips; - rowServiceEp->getIpText(ips); + rowServiceEp->getHostText(ips); rowServiceSock.setown(ISocket::create_ip(rowServiceEp->port, ips.str())); } @@ -5662,7 +5662,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface { #ifdef _DEBUG sock->getPeerEndpoint(eps); - eps.getUrlStr(peerURL); + eps.getEndpointHostText(peerURL); PROGLOG("Server accepting from %s", peerURL.str()); #endif /* NB: if it hits the thread pool limit, it will start throttling (introducing delays), @@ -5676,7 +5676,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface { #ifdef _DEBUG sockSSL->getPeerEndpoint(eps); - eps.getUrlStr(peerURL.clear()); + eps.getEndpointHostText(peerURL.clear()); PROGLOG("Server accepting SECURE from %s", peerURL.str()); #endif runClient(sockSSL.getClear(), false); @@ -5686,7 +5686,7 @@ class CRemoteFileServer : implements IRemoteFileServer, public CInterface { #ifdef _DEBUG acceptedRSSock->getPeerEndpoint(eps); - eps.getUrlStr(peerURL.clear()); + eps.getEndpointHostText(peerURL.clear()); PROGLOG("Server accepting row service socket from %s", peerURL.str()); #endif runClient(acceptedRSSock.getClear(), true); @@ -5987,7 +5987,7 @@ class RemoteFileSlowTest : public CppUnit::TestFixture basePath.append("//"); SocketEndpoint ep(serverPort); - ep.getUrlStr(basePath); + ep.getEndpointHostText(basePath); char cpath[_MAX_DIR]; if (!GetCurrentDirectory(_MAX_DIR, cpath)) diff --git a/fs/dafsstream/dafsstream.cpp b/fs/dafsstream/dafsstream.cpp index cefaf24b59f..f0c1ea740be 100644 --- a/fs/dafsstream/dafsstream.cpp +++ b/fs/dafsstream/dafsstream.cpp @@ -262,7 +262,7 @@ class CDFUFile : public CSimpleInterfaceOf, implements IDFUFileA for (unsigned c=0; cqueryNode(p, c)->endpoint().getUrlStr(host); + fileDesc->queryNode(p, c)->endpoint().getEndpointHostText(host); unsigned pos = p*maxCopiesPerPart+c; if (hosts.size() <= pos) hosts.resize(pos+1); // ensure big enough @@ -562,13 +562,13 @@ class CDaFileSrvClientBase : public CInterfaceOf if (0 == serverVersion) { StringBuffer str; - throwDsFsClientExceptionV(DaFsClient_ConnectionFailure, "CDaFileSrvClientBase: Failed to connect to %s", daFsConnection->queryEp().getUrlStr(str).str()); + throwDsFsClientExceptionV(DaFsClient_ConnectionFailure, "CDaFileSrvClientBase: Failed to connect to %s", daFsConnection->queryEp().getEndpointHostText(str).str()); } if (serverVersion < DAFILESRV_STREAMREAD_MINVERSION) { StringBuffer str; - throwDsFsClientExceptionV(DaFsClient_TooOld, "CDaFileSrvClientBase: server ersion(%u), too old connect to %s", serverVersion, daFsConnection->queryEp().getUrlStr(str).str()); + throwDsFsClientExceptionV(DaFsClient_TooOld, "CDaFileSrvClientBase: server ersion(%u), too old connect to %s", serverVersion, daFsConnection->queryEp().getEndpointHostText(str).str()); } } void start() @@ -1314,7 +1314,7 @@ class CDFUPartFlatWriter : public CDFUPartWriterBase e->Release(); } StringBuffer msg; - daFsConnection->queryEp().getUrlStr(msg); + daFsConnection->queryEp().getEndpointHostText(msg); WARNLOG("Stream writing not supported by dafilesrv(%s), attempting unsecured direct connection", msg.str()); RemoteFilename rfn; file->queryFileDescriptor().getFilename(part, 0, rfn); diff --git a/helm/examples/certmanager/README-vault-pki.md b/helm/examples/certmanager/README-vault-pki.md deleted file mode 100644 index 78837f751f6..00000000000 --- a/helm/examples/certmanager/README-vault-pki.md +++ /dev/null @@ -1,361 +0,0 @@ -# Install Hashicorp Vault - -See also https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager for a more Vault centric tutorial on setting up cert-manager with vault. - -## Add hashicorp to you helm repo -```bash -helm repo add hashicorp https://helm.releases.hashicorp.com -``` - -## Helm install hashicorp vault - -Disable the vault sidecar injector by setting "injector.enabled=false". - -```bash -helm install vault hashicorp/vault --set "injector.enabled=false" -``` - -Check the pods: -```bash -kubectl get pods -``` - -Vault pods should be running, but not ready - -```bash -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -vault-0 0/1 Running 0 6s -``` - -## Initialize and unseal the vault - -Initialize Vault with one key share and one key threshold. Saving off the output in json format so -we can utilize the unseal key and root token later. - -```bash -kubectl exec vault-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > init-keys.json -``` - -View the unseal key found in init-keys.json. - -```bash -cat init-keys.json | jq -r ".unseal_keys_b64[]" -``` - -Create an environment variable holding the unseal key: - -```bash -VAULT_UNSEAL_KEY=$(cat init-keys.json | jq -r ".unseal_keys_b64[]") -``` - -Unseal Vault running on the vault-0 pod with the $VAULT_UNSEAL_KEY. - -```bash -kubectl exec vault-0 -- vault operator unseal $VAULT_UNSEAL_KEY -``` - -Check the pods: -```bash -kubectl get pods -``` - -Vault pods should now be running and ready. - -## Configure the Vault PKI secrets engine (certificate authority) - -View the vault root token: -```bash -cat init-keys.json | jq -r ".root_token" -``` - -Create a variable named VAULT_ROOT_TOKEN to capture the root token. -```bash -VAULT_ROOT_TOKEN=$(cat init-keys.json | jq -r ".root_token") -``` - -Login to Vault running on the vault-0 pod with the $VAULT_ROOT_TOKEN. -```bash -kubectl exec vault-0 -- vault login $VAULT_ROOT_TOKEN -``` - -Start an interactive shell session on the vault-0 pod. -```bash -kubectl exec --stdin=true --tty=true vault-0 -- /bin/sh -``` -We are now working from the vault-0 pod. You should see a prompt, something like: - -```bash -/ $ -``` - -Enable the PKI secrets engine at its default path. -```bash -vault secrets enable pki -``` - -Configure the max lease time-to-live (TTL) to 8760h. -```bash -vault secrets tune -max-lease-ttl=8760h pki -``` - -# Vault CA key pair - -Vault can accept an existing key pair, or it can generate its own self-signed root. In general, they recommend maintaining your root CA outside of Vault and providing Vault a signed intermediate CA, but for this demo we will keep it simple and generate a self signed root certificate. - -Generate a self-signed certificate valid for 8760h. -```bash -vault write pki/root/generate/internal common_name=example.com ttl=8760h -``` - -Configure the PKI secrets engine certificate issuing and certificate revocation list (CRL) endpoints to use the Vault service in the default namespace. -```bash -vault write pki/config/urls issuing_certificates="http://vault.default:8200/v1/pki/ca" crl_distribution_points="http://vault.default:8200/v1/pki/crl" -``` - -For our local MTLS certificates we will use our kubernetes namespace as our domain name. This will allow us to recongize where these components reside. -For our public TLS certificates for this demo we will use myhpcc.com as our domain. - -Configure a role named hpccnamespace that enables the creation of certificates hpccnamespace domain with any subdomains. - -```bash -vault write pki/roles/hpcclocal key_type=any allowed_domains=default allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72h -``` - -Configure a role named myhpcc-dot-com that enables the creation of certificates myhpcc.com domain with any subdomains. - -```bash -vault write pki/roles/myhpcc-dot-com allowed_domains=myhpcc.com allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72h -``` - -Create a policy named pki that enables read access to the PKI secrets engine paths. - -```bash -vault policy write pki - < - - GET/WsSmc/HttpEchoname=doe,joe&number=1
Accept-Encoding: gzip, deflate
Accept: text/xml
 - - -``` diff --git a/helm/examples/certmanager/values-vault-pki.yaml b/helm/examples/certmanager/values-vault-pki.yaml deleted file mode 100644 index 0f44fdfcd2d..00000000000 --- a/helm/examples/certmanager/values-vault-pki.yaml +++ /dev/null @@ -1,33 +0,0 @@ -certificates: - issuers: - local: - spec: - # delete any alternative spec config, and then add vault - ca: null - selfSigned: null - vault: - server: http://vault.default:8200 - path: pki/sign/hpcclocal - auth: - kubernetes: - mountPath: /v1/auth/kubernetes - role: issuer - secretRef: - name: tbd # requires service account secret, set from command line - key: token - public: - domain: myhpcc.com - spec: - # delete any alternative spec config, and then add vault - ca: null - selfSigned: null - vault: - server: http://vault.default:8200 - path: pki/sign/myhpcc-dot-com - auth: - kubernetes: - mountPath: /v1/auth/kubernetes - role: issuer - secretRef: - name: tbd # requires service account secret, set from command line - key: token diff --git a/helm/examples/vault-pki-remote/README-vault-pki.md b/helm/examples/vault-pki-remote/README-vault-pki.md new file mode 100644 index 00000000000..c8c74e1e35c --- /dev/null +++ b/helm/examples/vault-pki-remote/README-vault-pki.md @@ -0,0 +1,237 @@ +# Using a Hashicorp Vault PKI Certificate Authority to establish trust between two HPCC environments + +This walkthough demonstrates using a single Hashicorp Vault PKI Certificate quthority to establish trust between two or more HPCC environments. + +In the case of this example each HPCC environment is in a separate kubernetes namespace. + +## Install hashicorp vault service in dev mode: + +This is for development only, never deploy this way in production. +Deploying in dev mode sets up an in memory kv store that won't persist secret values across restart, and the vault will automatically be unsealed. + +In dev mode the default root token is simply the string "root". + +Add Hashicorp helm repo: + +```bash +helm repo add hashicorp https://helm.releases.hashicorp.com +``` + +Update Helm repos. + +```bash +helm repo update +``` + +Install vault server. + +Note that a recent change to the developer mode vault means that you have to set the VAULT_DEV_LISTEN_ADDRESS environment variable as shown in order to access the vault service from an external pod. + +```bash +helm install vault hashicorp/vault --set "injector.enabled=false" --set "server.dev.enabled=true" --set 'server.extraEnvironmentVars.VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' --namespace vaultns --create-namespace +``` + +Check the pods: +```bash +kubectl get pods -n vaultns +``` + +Vault pods should now be running and ready. + + +## Setting up vault + +Tell the vault command line application the server location (dev mode is http, default location is https) + +```bash +export VAULT_ADDR=http://127.0.0.1:8200 +``` + +Export an environment variable for the vault CLI to authenticate with the Vault server. Because we installed dev mode, the vault token is 'root'. + +```bash +export VAULT_TOKEN=root +``` + +In a separate terminal window start vault port forwarding. + +```bash +kubectl port-forward vault-0 8200:8200 -n vaultns +``` + +Login to the vault command line using the vault root token (development mode defaults to "root"): + +```bash +vault login root +``` + +## Enable the PKI secrets engine at its default path. +```bash +vault secrets enable pki +``` + +Configure the max lease time-to-live (TTL) to 8760h. +```bash +vault secrets tune -max-lease-ttl=87600h pki +``` + +Generate the hpcc remote issuer CA, give it an issuer name. + +```bash +vault write -field=certificate pki/root/generate/internal common_name="hpcc-issuer" issuer_name="hpcc-remote-issuer" ttl=87600h +``` + +Configure the PKI secrets engine certificate issuing and certificate revocation list (CRL) endpoints to use the Vault service in the "vaultns" namespace. + +If you installed vault into a different namespace update the urls, replacing "vaultns" with the namespace used. + +```bash +vault write pki/config/urls issuing_certificates="http://vault.vaultns:8200/v1/pki/ca" crl_distribution_points="http://vault.vaultns:8200/v1/pki/crl" +``` + +For our local MTLS certificates we will use our kubernetes namespace as our domain name. This will allow us to recongize where these components reside. +For our public TLS certificates for this demo we will use myhpcc.com as our domain. + +Configure a role named hpccnamespace that enables the creation of certificates hpccnamespace domain with any subdomains. + +```bash +vault write pki/roles/hpccremote key_type=any allowed_domains="hpcc1,hpcc2" allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72 +``` + +Create a policy named pki that enables read access to the PKI secrets engine paths. + +```bash +vault policy write hpcc-remote-pki - <- {{ $check_cmd.command }}; + exitCode=$?; k8s_postjob_clearup.sh; -{{- if $misc.postJobCommandViaSidecar -}} ; - touch /wait-and-run/{{ .me.name }}.jobdone -{{- else if $postJobCommand -}} ; - {{ $postJobCommand }} +{{- if $misc.postJobCommandViaSidecar -}} + touch /wait-and-run/{{ .me.name }}.jobdone; +{{- else if $postJobCommand -}} + {{ $postJobCommand }} ; {{- end }} + exit $exitCode; {{- end -}} -{{/* -Use cert-manager to create a public certificate and private key for use with TLS -There are separate certificate issuers for local and public certificates -by default public certificates are self-signed and local certificates are signed -by our own certificate authority. A CA certificate is also provided to the pod -so that we can recognize the signature of our own CA. -NB: if optional 'issuer' passed in use it, otherwise base on visibility and -use "public" or "local" -*/}} -{{- define "hpcc.addCertificate" }} +{{- define "hpcc.addCertificateImpl" }} {{- if (.root.Values.certificates | default dict).enabled -}} - {{- $externalCert := ((hasKey . "external") | ternary .external (ne (include "hpcc.isVisibilityPublic" .) "")) -}} - {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}} - {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} + {{- $externalCert := .externalCert -}} + {{- $issuerKeyName := .issuerKeyName -}} {{- if eq (include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName)) "true" -}} {{- $issuer := get .root.Values.certificates.issuers $issuerKeyName -}} {{- if $issuer -}} @@ -1754,9 +1790,33 @@ spec: {{- end -}} {{- end -}} +{{/* +Use cert-manager to create a public certificate and private key for use with TLS +There are separate certificate issuers for local and public certificates +by default public certificates are self-signed and local certificates are signed +by our own certificate authority. A CA certificate is also provided to the pod +so that we can recognize the signature of our own CA. +NB: if optional 'issuer' passed in use it, otherwise base on visibility and +use "public" or "local" +*/}} +{{- define "hpcc.addCertificate" }} + {{- if (.root.Values.certificates | default dict).enabled -}} + {{- $externalCert := ((hasKey . "external") | ternary .external (ne (include "hpcc.isVisibilityPublic" .) "")) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" . )) -}} + {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} + {{- $_ := set . "externalCert" $externalCert -}} + {{- $_ := set . "issuerKeyName" $issuerKeyName -}} + {{- include "hpcc.addCertificateImpl" . -}} + {{- if and (.includeRemote) (ne "remote" $issuerKeyName) -}} + {{- $_ := set . "issuerKeyName" "remote" -}} + {{- include "hpcc.addCertificateImpl" . -}} + {{- end -}} + {{- end -}} +{{- end -}} + {{/* Builds the commonName for a client certificate. Used in creation of both certificate and access control list. - Pass in root, client (name), instance (myeclwatch), component (eclwatch), visibility, external (bool, optional) + Pass in root, client (name), instance (myeclwatch), visibility, external (bool, optional) */}} {{- define "hpcc.getClientCommonName" -}} {{- if (.root.Values.certificates | default dict).enabled -}} @@ -1779,20 +1839,39 @@ Builds the commonName for a client certificate. Used in creation of both certif {{- end -}} {{/* -Turns an array of remoteClients into a | delimited string to be used for the trusted_peers element of SecureSocket settings. - Pass in root, remoteClients, instance (myeclwatch), component (eclwatch), visibility +Turns arrays of trustClients and remoteClients into a | delimited string to be used for the trusted_peers element of SecureSocket settings. + Pass in root, trustClients, remoteClients, instance (myeclwatch), visibility */}} {{- define "hpcc.getTrustedPeerString" -}} - {{- if not (hasKey . "remoteClients") -}} + {{- if not (or (hasKey . "remoteClients") (hasKey . "trustClients")) -}} anyone {{- else -}} {{/* Turn remoteClients array into one single array element which is a | delimited string */}} {{- $instance := .instance -}} - {{- $component := .component -}} {{- $visibility := .visibility -}} {{- $root := .root -}} {{- range $remoteClient := .remoteClients -}} - {{- include "hpcc.getClientCommonName" (dict "root" $root "client" $remoteClient.name "instance" $instance "component" $component "visibility" $visibility "issuerKeyName" "remote") -}}| + {{- include "hpcc.getClientCommonName" (dict "root" $root "client" $remoteClient.name "instance" $instance "visibility" $visibility "issuerKeyName" "remote") -}}| + {{- end -}} + {{- range $trustClient := .trustClients -}} + {{- $trustClient.commonName -}}| + {{- end -}} + {{- if .incluedRoxieAndEspServices -}} + {{- $allowedESPs := list "eclwatch" "eclservices" "eclqueries" -}} + {{- $remoteIssuer := get $root.Values.certificates.issuers "remote" -}} + {{- if and ($remoteIssuer) (hasKey $remoteIssuer "domain") -}} + {{- $domain := $remoteIssuer.domain -}} + {{- range $esp := $root.Values.esp -}} + {{- if has $esp.application $allowedESPs -}} + {{- $esp.name -}}.{{- $domain -}}| + {{- end -}} + {{- end -}} + {{- range $roxie := $root.Values.roxie -}} + {{- range $roxieService := $roxie.services -}} + {{- $roxieService.name -}}.{{- $domain -}}| + {{- end -}} + {{- end -}} + {{- end -}} {{- end -}} {{- end -}} {{- end }} @@ -1814,7 +1893,7 @@ Will create a TLS based access control list which ESP will check to make sure a Pass in root, client (name), organization (optional), instance (myeclwatch), component (eclwatch), visibility, secretTemplate (optional) */}} -{{- define "hpcc.addClientCertificate" }} +{{- define "hpcc.addExternalRemoteClientCertificate" }} {{- if (.root.Values.certificates | default dict).enabled -}} {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}} {{- $issuerKeyName := .issuerKeyName | default (ternary "remote" "local" $externalCert) -}} @@ -1933,11 +2012,11 @@ spec: {{/* Add a certficate volume mount for a component NB: if optional 'issuer' passed in use it, otherwise base on visibility and -use "public" or "local" +use "public" or "local" */}} -{{- define "hpcc.addCertificateVolumeMount" -}} +{{- define "hpcc.addCertificateVolumeMountImpl" -}} {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}} - {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" . )) -}} {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} {{- /* A .certificate parameter means the user explicitly configured a certificate to use @@ -1965,15 +2044,30 @@ use "public" or "local" {{- end -}} {{- end -}} +{{- define "hpcc.addCertificateVolumeMount" }} + {{- if (.root.Values.certificates | default dict).enabled -}} + {{- $externalCert := ((hasKey . "external") | ternary .external (ne (include "hpcc.isVisibilityPublic" .) "")) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" . )) -}} + {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} + {{- $_ := set . "externalCert" $externalCert -}} + {{- $_ := set . "issuerKeyName" $issuerKeyName -}} + {{- include "hpcc.addCertificateVolumeMountImpl" . -}} + {{- if and (.includeRemote) (ne "remote" $issuerKeyName) -}} + {{- $_ := set . "issuerKeyName" "remote" -}} + {{- include "hpcc.addCertificateVolumeMountImpl" . -}} + {{- end -}} + {{- end -}} +{{- end -}} + + {{/* Add a secret volume for a certificate NB: if optional 'issuer' passed in use it, otherwise base on visibility and -use "public" or "local" +use "public" or "local" */}} -{{- define "hpcc.addCertificateVolume" -}} - {{- $externalCert := or (and (hasKey . "external") .external) (ne (include "hpcc.isVisibilityPublic" .) "") -}} - {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" . )) -}} - {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} +{{- define "hpcc.addCertificateVolumeImpl" -}} + {{- $externalCert := .externalCert -}} + {{- $issuerKeyName := .issuerKeyName -}} {{- /* A .certificate parameter means the user explicitly configured a certificate to use otherwise check if certificate generation is enabled @@ -2003,6 +2097,30 @@ use "public" or "local" {{- end -}} {{- end -}} +{{- define "hpcc.addCertificateVolume" }} + {{- if (.root.Values.certificates | default dict).enabled -}} + {{- $externalCert := ((hasKey . "external") | ternary .external (ne (include "hpcc.isVisibilityPublic" .) "")) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" . )) -}} + {{- $issuerKeyName := .issuerKeyName | default (ternary $externalIssuerKeyName "local" $externalCert) -}} + {{- $_ := set . "externalCert" $externalCert -}} + {{- $_ := set . "issuerKeyName" $issuerKeyName -}} + {{- include "hpcc.addCertificateVolumeImpl" . -}} + {{- if and (.includeRemote) (ne "remote" $issuerKeyName) -}} + {{- $_ := set . "issuerKeyName" "remote" -}} + {{- include "hpcc.addCertificateVolumeImpl" . -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{- define "hpcc.addRemoteCertificateVolume" }} + {{- if (.root.Values.certificates | default dict).enabled -}} + {{- $externalCert := ((hasKey . "external") | ternary .external (ne (include "hpcc.isVisibilityPublic" .) "")) -}} + {{- $_ := set . "externalCert" $externalCert -}} + {{- $_ := set . "issuerKeyName" "remote" -}} + {{- include "hpcc.addCertificateVolumeImpl" . -}} + {{- end -}} +{{- end -}} + {{/* Add the certificate volume mount for a roxie udp key */}} diff --git a/helm/hpcc/templates/_warnings.tpl b/helm/hpcc/templates/_warnings.tpl index e07132d3bd5..70e590c7b4e 100644 --- a/helm/hpcc/templates/_warnings.tpl +++ b/helm/hpcc/templates/_warnings.tpl @@ -145,7 +145,7 @@ Pass in dict with root and warnings {{- if not $espservice.tls -}} {{- if (and ($ctx.root.Values.certificates|default false) $ctx.root.Values.certificates.enabled) -}} {{- $externalCert := (ne (include "hpcc.isVisibilityPublic" (dict "root" $ctx.root "visibility" $espservice.service.visibility)) "") -}} - {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" $espservice )) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" $espservice )) -}} {{- $issuerKeyName := ternary $externalIssuerKeyName "local" $externalCert -}} {{- if ne (include "hpcc.isIssuerEnabled" (dict "root" $ctx.root "issuerKeyName" "local")) "true" -}} {{- $_ := set $ctx "TLSdisabled" (append $ctx.TLSdisabled $espservice.name) -}} diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml index 4fa39f79ccd..259c90ce7c9 100644 --- a/helm/hpcc/templates/dafilesrv.yaml +++ b/helm/hpcc/templates/dafilesrv.yaml @@ -105,7 +105,7 @@ kind: ConfigMap {{- $_ := set $service "labels" (merge $service.labels (dict "server" .name)) -}} {{ include "hpcc.addService" ( dict "root" $ "name" .name "service" $service "selector" .name "defaultPort" 7600 ) }} --- -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" "dafilesrv" "visibility" .service.visibility) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" "dafilesrv" "visibility" .service.visibility "trustClients" .trustClients) }} {{/* Generate network policies for spray services to allow ingress from dfuservers */}} diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml index 56d77eb3000..3dad386a2c4 100644 --- a/helm/hpcc/templates/eclagent.yaml +++ b/helm/hpcc/templates/eclagent.yaml @@ -92,7 +92,7 @@ data: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} {{ include "hpcc.addVolumeMounts" . | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" $apptype) | indent 12 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" $apptype "includeRemote" true) | indent 12 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }} {{- end }} @@ -100,7 +100,7 @@ data: {{ include "hpcc.addConfigMapVolume" .me | indent 10 }} {{ include "hpcc.addVolumes" . | indent 10 }} {{ include "hpcc.addSecretVolumes" . | indent 10 }} -{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" $apptype) | indent 10 }} +{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" $apptype "includeRemote" true) | indent 10 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }} {{- end }} @@ -174,12 +174,13 @@ spec: {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "eclagent") | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" $apptype "includeRemote" true) | indent 8 }} volumes: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclagent") | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" $apptype "includeRemote" true) | indent 6 }} + --- {{- if and (hasKey . "hpa") .useChildProcesses }} {{- include "hpcc.addHorizontalPodAutoscaler" (dict "name" .name "kind" "Deployment" "hpa" .hpa) }} @@ -188,8 +189,7 @@ spec: kind: ConfigMap {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.agentConfigMap")) }} --- -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclagent") }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" $apptype) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" $apptype "includeRemote" true) }} --- {{- $egressedPodLabels := list (ternary .name (printf "%s-job" .name) (.useChildProcesses | default false)) -}} {{- include "hpcc.addEgress" (dict "root" $ "me" . "labels" $egressedPodLabels) | nindent 0 -}} diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml index 016edae359d..af84ec9d1a4 100644 --- a/helm/hpcc/templates/eclccserver.yaml +++ b/helm/hpcc/templates/eclccserver.yaml @@ -91,7 +91,7 @@ data: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} {{ include "hpcc.addVolumeMounts" . | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "compile") | indent 12 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "compile" "includeRemote" true) | indent 12 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }} {{- end }} @@ -99,7 +99,7 @@ data: {{ include "hpcc.addConfigMapVolume" .me | indent 10 }} {{ include "hpcc.addVolumes" . | indent 10 }} {{ include "hpcc.addSecretVolumes" . | indent 10 }} -{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "compile") | indent 10 }} +{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "compile" "includeRemote" true) | indent 10 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }} {{- end }} @@ -183,14 +183,14 @@ spec: {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 -}} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "eclccserver") | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" .name "component" "eclccserver" "includeRemote" true) | indent 8 }} - name: "hpccbundles" mountPath: "/home/hpcc/.HPCCSystems" volumes: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclccserver") | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" .name "component" "eclccserver" "includeRemote" true) | indent 6 }} - name: hpccbundles emptyDir: {} --- @@ -201,8 +201,8 @@ spec: kind: ConfigMap {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.eclccServerConfigMap")) }} --- -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclccserver") }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "compile") }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclccserver" "includeRemote" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "compile" "includeRemote" true) }} --- {{- include "hpcc.addEgress" (dict "root" $ "me" . "labels" (list .name (printf "%s-job" .name))) | nindent 0 -}} {{- end }} diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml index e49c0467bb2..8b3c1cbe8a6 100644 --- a/helm/hpcc/templates/esp.yaml +++ b/helm/hpcc/templates/esp.yaml @@ -38,7 +38,7 @@ data: {{- include "hpcc.generateMetricsConfig" . | indent 6 }} {{- if and .root.Values.certificates .root.Values.certificates.enabled }} {{- $externalCert := (ne (include "hpcc.isVisibilityPublic" (dict "root" .root "visibility" .me.service.visibility)) "") -}} - {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteClientCertificates" .me )) -}} + {{- $externalIssuerKeyName := ternary "remote" "public" (eq "true" ( include "hpcc.usesRemoteIssuer" .me )) -}} {{- $issuerKeyName := ternary $externalIssuerKeyName "local" $externalCert -}} {{- if not (hasKey .me "tls" )}} tls: {{ include "hpcc.isIssuerEnabled" (dict "root" .root "issuerKeyName" $issuerKeyName) }} @@ -47,12 +47,12 @@ data: {{- if $externalCert }} certificate: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.crt privatekey: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/tls.key - {{- if (hasKey .me "remoteClients" )}} + {{- if or (hasKey .me "remoteClients" ) (hasKey .me "trustClients" )}} verify: enable: true address_match: false accept_selfsigned: false - trusted_peers: [ {{ include "hpcc.getTrustedPeerString" (dict "root" .root "remoteClients" .me.remoteClients "instance" .me.name "component" .me.application "visibility" .me.service.visibility) | quote }} ] + trusted_peers: [ {{ include "hpcc.getTrustedPeerString" (dict "root" .root "remoteClients" .me.remoteClients "trustClients" .me.trustClients "instance" .me.name "visibility" .me.service.visibility) | quote }} ] ca_certificates: path: /opt/HPCCSystems/secrets/certificates/{{ $issuerKeyName }}/ca.crt {{- end }} @@ -158,7 +158,7 @@ spec: {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients "trustClients" .trustClients "includeRemote" true) | indent 8 }} {{- if $commonCtx.externalCert }} {{- include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" $application "name" .name "external" false) | nindent 8 }} {{- end }} @@ -169,7 +169,7 @@ spec: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "certificate" .certificate "visibility" .service.visibility "remoteClients" .remoteClients "trustClients" .trustClients "includeRemote" true) | indent 6 }} {{- if $commonCtx.externalCert }} {{- include "hpcc.addCertificateVolume" (dict "root" $ "component" $application "name" .name "external" false) | nindent 6 }} {{- end }} @@ -191,13 +191,13 @@ kind: ConfigMap {{- $_ := set $service "labels" dict -}} {{- end -}} {{- $_ := set $service "labels" (merge $service.labels (dict "server" $application)) -}} -{{ include "hpcc.addService" ( dict "root" $ "name" .name "service" $service "selector" .name "defaultPort" 8880 "remoteClients" .remoteClients) }} +{{ include "hpcc.addService" ( dict "root" $ "name" .name "service" $service "selector" .name "defaultPort" 8880 "remoteClients" .remoteClients "trustClients" .trustClients) }} --- {{ include "hpcc.addEgress" $commonCtx }} {{- end }} {{- end }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "visibility" .service.visibility "remoteClients" .remoteClients) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "visibility" .service.visibility "remoteClients" .remoteClients "trustClients" .trustClients "includeRemote" true) }} {{- if $commonCtx.externalCert }} {{- include "hpcc.addCertificate" (dict "root" $ "name" .name "service" .service "component" $application "external" false) }} {{- end }} @@ -208,7 +208,7 @@ kind: ConfigMap {{- $instance := .name -}} {{- $visibility := .service.visibility -}} {{- range $remoteClient := .remoteClients }} - {{ include "hpcc.addClientCertificate" (dict "root" $ "client" $remoteClient.name "organization" $remoteClient.organization "instance" $instance "component" $application "visibility" $visibility "secretTemplate" $remoteClient.secretTemplate) }} + {{ include "hpcc.addExternalRemoteClientCertificate" (dict "root" $ "client" $remoteClient.name "organization" $remoteClient.organization "instance" $instance "component" $application "visibility" $visibility "secretTemplate" $remoteClient.secretTemplate) }} {{- end }} {{- end }} {{- end }} diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml index df0d4b6fda7..32139e9699c 100644 --- a/helm/hpcc/templates/localroxie.yaml +++ b/helm/hpcc/templates/localroxie.yaml @@ -120,14 +120,14 @@ spec: {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{- include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 8 }} {{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 8 }} volumes: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" false) | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localroxie" "external" true "includeRemote" true) | indent 6 }} {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "name" $roxie.name "component" "localudpkey" ) | indent 6 }} --- {{- range $service := $roxie.services }} @@ -140,7 +140,7 @@ kind: ConfigMap {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.localroxieConfigMap")) }} --- {{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" false) }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $roxie.name "services" $roxie.services "component" "localroxie" "external" true "includeRemote" true) }} {{ include "hpcc.addUDPCertificate" (dict "root" $ "name" $roxie.name "component" "localudpkey") }} --- {{ include "hpcc.addEgress" $commonCtx }} diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml index 206ecf66f63..5b07350ba5d 100644 --- a/helm/hpcc/templates/roxie.yaml +++ b/helm/hpcc/templates/roxie.yaml @@ -41,7 +41,7 @@ data: - name: {{ $service.name }} {{ toYaml (omit $service "tls" "name") | indent 8 }} {{- if ne (int $service.servicePort) 0 }} - {{- include "hpcc.addTLSServiceEntries" (dict "root" $root "service" $service "component" $component "visibility" $service.visibility) | indent 6 }} + {{- include "hpcc.addTLSServiceEntries" (dict "root" $root "service" $service "component" $component "visibility" $service.visibility "remoteClients" $service.remoteClients "trustClients" $service.trustClients "includeTrustedPeers" true "incluedRoxieAndEspServices" true) | indent 6 }} {{- end }} {{- end }} {{ toYaml ( omit .me "logging" "topoServer" "encryptInTransit" "env" "services") | indent 6 }} @@ -100,6 +100,9 @@ data: {{- $servername := printf "%s-server" $roxie.name -}} {{- $udpkeyname := $roxie.name -}} {{- range $service := $roxie.services }} + {{- range $remoteClient := $service.remoteClients }} + {{ include "hpcc.addExternalRemoteClientCertificate" (dict "root" $ "client" $remoteClient.name "organization" $remoteClient.organization "instance" $service.name "component" "roxie" "visibility" $service.visibility "secretTemplate" $remoteClient.secretTemplate) }} + {{- end }} {{- if ne (int $service.servicePort) 0 }} {{- $_ := set $service "port" $service.servicePort }} {{- end }} @@ -301,24 +304,24 @@ spec: {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 8 }} {{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "component" "udpkey" "name" $udpkeyname ) | indent 8 }} volumes: {{ include "hpcc.addConfigMapVolume" $roxie | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "external" false) | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-server" "name" $servername "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }} {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "component" "udpkey" "name" $udpkeyname) | indent 6 }} --- {{ include "hpcc.addCertificate" (dict "root" $ "name" $servername "services" $roxie.services "component" "roxie-server" "external" false) }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" $servername "services" $roxie.services "component" "roxie-server" "external" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $servername "services" $roxie.services "component" "roxie-server" "external" true "includeRemote" true) }} --- {{ end -}} {{- $agentPublicCertName := printf "%s-agent" $roxie.name }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" $agentPublicCertName "services" $roxie.services "component" "roxie-agent" "external" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $agentPublicCertName "services" $roxie.services "component" "roxie-agent" "external" true "includeRemote" true) }} {{ range $c, $e := until ($commonCtx.numChannels|int) -}} {{- $channel := add $c 1 -}} @@ -411,7 +414,7 @@ spec: {{- if not $roxie.serverReplicas }} {{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true) | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 8 }} {{ include "hpcc.addUDPCertificateVolumeMount" (dict "root" $ "component" "udpkey" "name" $udpkeyname ) | indent 8 }} {{- end }} @@ -421,7 +424,7 @@ spec: {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} {{- if not $roxie.serverReplicas }} {{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $name "external" false) | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true) | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "component" "roxie-agent" "name" $agentPublicCertName "certificate" $roxie.certificate "external" true "includeRemote" true) | indent 6 }} {{ include "hpcc.addUDPCertificateVolume" (dict "root" $ "component" "udpkey" "name" $udpkeyname) | indent 6 }} {{- end }} --- diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml index f4bf7f7f111..57c4c01096b 100644 --- a/helm/hpcc/templates/thor.yaml +++ b/helm/hpcc/templates/thor.yaml @@ -117,7 +117,7 @@ data: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} {{ include "hpcc.addVolumeMounts" . | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "eclagent") | indent 12 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "eclagent" "includeRemote" true) | indent 12 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }} {{- end }} @@ -125,7 +125,7 @@ data: {{ include "hpcc.addConfigMapVolume" .me | indent 10 }} {{ include "hpcc.addVolumes" . | indent 10 }} {{ include "hpcc.addSecretVolumes" . | indent 10 }} -{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "eclagent") | indent 10 }} +{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "eclagent" "includeRemote" true) | indent 10 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }} {{- end }} @@ -184,7 +184,7 @@ data: {{ include "hpcc.addConfigMapVolumeMount" .me | indent 12 }} {{ include "hpcc.addVolumeMounts" . | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" . | indent 12 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "thormanager") | indent 12 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" .root "name" .me.name "component" "thormanager" "includeRemote" true) | indent 12 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolumeMount" . | indent 12 }} {{- end }} @@ -192,7 +192,7 @@ data: {{ include "hpcc.addConfigMapVolume" .me | indent 10 }} {{ include "hpcc.addVolumes" . | indent 10 }} {{ include "hpcc.addSecretVolumes" . | indent 10 }} -{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thormanager") | indent 10 }} +{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thormanager" "includeRemote" true) | indent 10 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }} {{- end }} @@ -251,9 +251,9 @@ data: workingDir: /var/lib/HPCCSystems volumeMounts: {{ include "hpcc.addConfigMapVolumeMount" $configCtx.me | indent 12 }} -{{ include "hpcc.addVolumeMounts" $configCtx | indent 12 }} +{{ include "hpcc.addVolumeMounts" (deepCopy $configCtx | merge (dict "id" (toString $containerNum))) | indent 12 }} {{ include "hpcc.addSecretVolumeMounts" $configCtx | indent 12 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $configCtx.root "name" $configCtx.me.name "component" "thorworker") | indent 12 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $configCtx.root "name" $configCtx.me.name "component" "thorworker" "includeRemote" true) | indent 12 }} {{- if and ($misc.postJobCommandViaSidecar) (eq $containerNum 1) }} {{ include "hpcc.addWaitAndRunVolumeMount" $configCtx | indent 12 }} {{- end }} @@ -262,7 +262,7 @@ data: {{ include "hpcc.addConfigMapVolume" .me | indent 10 }} {{ include "hpcc.addVolumes" . | indent 10 }} {{ include "hpcc.addSecretVolumes" . | indent 10 }} -{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thorworker") | indent 10 }} +{{ include "hpcc.addCertificateVolume" (dict "root" .root "name" .me.name "component" "thorworker" "includeRemote" true) | indent 10 }} {{- if $misc.postJobCommandViaSidecar }} {{ include "hpcc.addWaitAndRunVolume" . | indent 10 }} {{- end }} @@ -389,12 +389,12 @@ spec: {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent") | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent" "includeRemote" true) | indent 8 }} volumes: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent") | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent" "includeRemote" true) | indent 6 }} --- apiVersion: apps/v1 kind: Deployment @@ -450,27 +450,27 @@ spec: {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }} {{ include "hpcc.addVolumeMounts" $commonCtx | indent 8 }} {{ include "hpcc.addSecretVolumeMounts" $commonCtx | indent 8 }} -{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent") | indent 8 }} +{{ include "hpcc.addCertificateVolumeMount" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent" "includeRemote" true) | indent 8 }} volumes: {{ include "hpcc.addConfigMapVolume" . | indent 6 }} {{ include "hpcc.addVolumes" $commonCtx | indent 6 }} {{ include "hpcc.addSecretVolumes" $commonCtx | indent 6 }} -{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent") | indent 6 }} +{{ include "hpcc.addCertificateVolume" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent" "includeRemote" true) | indent 6 }} --- kind: ConfigMap {{ include "hpcc.generateConfig" ($commonCtx | merge (dict "configMapHelper" "hpcc.thorConfigMap")) }} --- -{{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent") }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent") }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.eclAgentName "component" "eclagent" "includeRemote" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" $commonCtx.thorAgentName "component" "thoragent" "includeRemote" true) }} ## thorworker and thormanager jobs generate pods with names based on wuid, the corresponding certificates will be named based on cluster and component. ## This wouldn't work for a service because the client couldn't identify the cert matched the fqdn used to connect, but will be good enough for our current purposes. -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "thormanager") }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "thorworker") }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "thormanager" "includeRemote" true) }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "thorworker" "includeRemote" true) }} {{- if $commonCtx.eclAgentUseChildProcesses }} -{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclagent") }} +{{ include "hpcc.addCertificate" (dict "root" $ "name" .name "component" "eclagent" "includeRemote" true) }} {{- end }} --- {{- $egressedPodLabels := list (printf "%s-thormanager-job" .name) (printf "%s-thorworker-job" .name) -}} diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json index db931992c85..5525f1916c2 100644 --- a/helm/hpcc/values.schema.json +++ b/helm/hpcc/values.schema.json @@ -544,6 +544,17 @@ "waitForMount": { "type": "boolean" }, + "expert": { + "type": "object", + "description": "Custom internal options usually reserved for internal testing", + "properties": { + "validatePlaneScript": { + "description": "a list of bash commands to run to validate the plane is healthy", + "type": "array", + "items": { "type": "string" } + } + } + }, "blockedFileIOKB": { "description": "Optimal block size for efficient reading from this plane. Implementations will use if they can", "type": "integer", @@ -557,6 +568,7 @@ "subPath": {}, "secret": {}, "pvc": {}, + "hostPath": {}, "hostGroup": {}, "hosts": {}, "umask": {}, @@ -584,6 +596,10 @@ "description": "optional name of any secret required to access this storage plane", "type": "string" }, + "hostPath": { + "description": "optional, this will create a hostPath volume (mutually exclusive with using pvc)", + "type": "string" + }, "pvc": { "description": "optional name of the persistent volume claim for this plane", "type": "string" @@ -689,6 +705,7 @@ "subPath": {}, "secret": {}, "pvc": {}, + "hostPath": {}, "hostGroup": {}, "hosts": {}, "umask": {}, @@ -1428,6 +1445,9 @@ "remoteClients": { "$ref": "#/definitions/remoteClients" }, + "trustClients": { + "$ref": "#/definitions/trustClients" + }, "corsAllowed": { "$ref": "#/definitions/corsAllowed" }, @@ -1507,6 +1527,21 @@ }, "hpa": { "$ref": "#/definitions/hpa" + }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" } } }, @@ -2189,6 +2224,21 @@ "minimum": 0, "description": "Interval (in milliseconds) between checks that client socket is still open" }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" + }, "expert": { "description": "Custom internal options usually reserved for internal testing", "type": "object" @@ -2277,6 +2327,12 @@ "tls": { "type": "boolean", "description": "Whether the roxie service uses tls. Requires cert-manager or custom certificate." + }, + "remoteClients": { + "$ref": "#/definitions/remoteClients" + }, + "trustClients": { + "$ref": "#/definitions/trustClients" } }, "required": [ "name", "servicePort" ], @@ -2426,6 +2482,21 @@ }, "allowedPipePrograms": { "$ref" : "#/definitions/allowedPipePrograms" + }, + "mapHttpCallUrlsToSecrets": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets" + }, + "warnIfUrlNotMappedToSecret": { + "type": "boolean", + "default": false, + "description": "In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets" + }, + "requireUrlsMappedToSecrets": { + "type": "boolean", + "default": false, + "description": "Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets" } } }, @@ -2996,6 +3067,18 @@ } } }, + "trustClients": { + "type": "array", + "items": { + "type": "object", + "properties": { + "commonName": { + "type": "string", + "description": "Trusted client common name" + } + } + } + }, "corsAllowed": { "type": "array", "items": { diff --git a/helm/hpcc/values.yaml b/helm/hpcc/values.yaml index 3633fa23537..ffb9c25f130 100644 --- a/helm/hpcc/values.yaml +++ b/helm/hpcc/values.yaml @@ -572,13 +572,21 @@ esp: # Add remote clients to generated client certificates and make the ESP require that one of the generated certificates is provided by a client in order to connect # When setting up remote clients make sure that certificates.issuers.remote.enabled is set to true. -# remoteClients: -# - name: petfoodApplicationProd -# organization: petfoodDept +# remoteClients: +# - name: petfoodapplicationprod +# organization: petfoodDept # secretTemplate: # annotations: # kubed.appscode.com/sync: "hpccenv=petfoodAppProd" # use kubed config-syncer to replicate certificate to namespace with matching annotation (also supports syncing with separate aks clusters) +# trustClients and remoteClients can be combined. Trust is far easier to manage and should now be the preferred mechanism. +# Trust is similar to remoteClients, but unlike remoteClients, the client certificates are generated elsewhere. +# If trust is present then esp will use mtls, with trust controlled by certificates.issuers.remote, which must be enabled. +# When using trustClients the remote issuer of each environment should point to the same certifate authority. +# Verification of identity is automatic if the CA matches, but only the clients listed here are actually allowed access +# trustClients: +# - commonName: rabc.example.com + service: ## port can be used to change the local port used by the pod. If omitted, the default port (8880) is used port: 8888 @@ -715,8 +723,20 @@ roxie: listenQueue: 200 numThreads: 30 visibility: local +# trustClients: +# - commonName: rabc.example.com +# - commonName: rbcd.example.com # Can override ingress rules for each service if desired - for example to add no additional ingress permissions you can use # ingress: [] + +# Trust is similar to remoteClients, but unlike remoteClients, the client certificates are generated elsewhere. +# If trust is present then roxie will use mtls with trust controlled by certificates.issuer.remote. +# Using the trust section the remote issuer of each environment should point to the same certifate authority. +# Verification of identity is automatic if the CA matches, but only the clients listed here are actually allowed access +# trust: +# - commonName: abc.example.com +# - commonName: bcd.example.com + ## replicas indicates the number of replicas per channel replicas: 2 numChannels: 2 diff --git a/initfiles/componentfiles/configxml/agentexec.xsl b/initfiles/componentfiles/configxml/agentexec.xsl index 770832c2552..b2e323b9ebc 100644 --- a/initfiles/componentfiles/configxml/agentexec.xsl +++ b/initfiles/componentfiles/configxml/agentexec.xsl @@ -99,6 +99,15 @@ + + + + + + + + + diff --git a/initfiles/componentfiles/configxml/eclagent_config.xsd.in b/initfiles/componentfiles/configxml/eclagent_config.xsd.in index 6721cfa38a4..d4c6e0b1153 100644 --- a/initfiles/componentfiles/configxml/eclagent_config.xsd.in +++ b/initfiles/componentfiles/configxml/eclagent_config.xsd.in @@ -286,6 +286,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/initfiles/componentfiles/configxml/roxie.xsd.in b/initfiles/componentfiles/configxml/roxie.xsd.in index 9d339c30e5d..3175574f6d7 100644 --- a/initfiles/componentfiles/configxml/roxie.xsd.in +++ b/initfiles/componentfiles/configxml/roxie.xsd.in @@ -673,6 +673,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/initfiles/componentfiles/configxml/thor.xsd.in b/initfiles/componentfiles/configxml/thor.xsd.in index 84e8fcb1133..b3556aa3324 100644 --- a/initfiles/componentfiles/configxml/thor.xsd.in +++ b/initfiles/componentfiles/configxml/thor.xsd.in @@ -656,6 +656,27 @@ + + + + In SOAPCALL and HTTPCALL check if URLs have been mapped to secrets + + + + + + + In SOAPCALL and HTTPCALL warn if URLs not mapped to secrets + + + + + + + Require SOAPCALL and HTTPCALL URLs are secrets or mapped to secrets + + + diff --git a/plugins/cassandra/cassandraembed.cpp b/plugins/cassandra/cassandraembed.cpp index 4718484a383..71cd1916954 100644 --- a/plugins/cassandra/cassandraembed.cpp +++ b/plugins/cassandra/cassandraembed.cpp @@ -127,7 +127,7 @@ void CassandraClusterSession::setOptions(const StringArray &options) if (contact_points[0]=='.') { SocketEndpoint ep(contact_points); - ep.getIpText(epText.clear()); + ep.getHostText(epText.clear()); contact_points = epText.str(); } } diff --git a/plugins/fileservices/fileservices.cpp b/plugins/fileservices/fileservices.cpp index 5f806431638..c3ca949b7ae 100644 --- a/plugins/fileservices/fileservices.cpp +++ b/plugins/fileservices/fileservices.cpp @@ -2320,7 +2320,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfRemotePull_impl(ICodeContext *ctx, if (wrap) req->setWrap(true); StringBuffer sourceDali; - queryCoven().queryComm().queryGroup().queryNode(0).endpoint().getUrlStr(sourceDali); + queryCoven().queryComm().queryGroup().queryNode(0).endpoint().getEndpointHostText(sourceDali); req->setSourceDali(sourceDali); req->setOverwrite(overwrite); req->setReplicate(replicate); @@ -2687,7 +2687,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsfResolveHostName(const char *hostna { StringBuffer ret; SocketEndpoint ep(hostname); - ep.getIpText(ret); + ep.getHostText(ret); return ret.detach(); } @@ -3044,7 +3044,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsGetEspURL(const char *username, cons if (streq(instanceAddress.str(),".")) { SocketEndpoint ep(instanceAddress.str()); - ep.getIpText(instanceAddress.clear()); + ep.getHostText(instanceAddress.clear()); } espURL.setf("%s://%s%s:%d", bindingProtocol.str(), credentials.str(), instanceAddress.str(), espBindingIter->query().getPropInt("@port",8010)); diff --git a/plugins/workunitservices/workunitservices.cpp b/plugins/workunitservices/workunitservices.cpp index 2ce9d76b417..6411b6d6d15 100644 --- a/plugins/workunitservices/workunitservices.cpp +++ b/plugins/workunitservices/workunitservices.cpp @@ -406,7 +406,7 @@ WORKUNITSERVICES_API void wsWorkunitList( } if (i+1>=sashaeps.ordinality()) { StringBuffer ips; - sashaeps.item(0).getIpText(ips); + sashaeps.item(0).getHostText(ips); throw MakeStringException(-1,"Time out to Sasha server on %s (server not running or query too complex)",ips.str()); } } diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp index 9c8d844a641..0b0ea57de7b 100644 --- a/roxie/ccd/ccdcontext.cpp +++ b/roxie/ccd/ccdcontext.cpp @@ -2686,7 +2686,7 @@ class CRoxieServerContext : public CRoxieContextBase, implements IRoxieServerCon WorkunitUpdate wu(&workUnit->lock()); wu->setDebugAgentListenerPort(debugEndpoint.port); //tells debugger what port to write commands to StringBuffer sb; - debugEndpoint.getIpText(sb); + debugEndpoint.getHostText(sb); wu->setDebugAgentListenerIP(sb); //tells debugger what IP to write commands to } options.timeLimit = 0; diff --git a/roxie/ccd/ccddali.cpp b/roxie/ccd/ccddali.cpp index 1d8d8936d3f..6465556dd74 100644 --- a/roxie/ccd/ccddali.cpp +++ b/roxie/ccd/ccddali.cpp @@ -734,7 +734,7 @@ class CRoxieDaliHelper : implements IRoxieDaliHelper, public CInterface first = false; else ret.append(','); - coven->query().endpoint().getUrlStr(ret); + coven->query().endpoint().getEndpointHostText(ret); } return ret; } diff --git a/roxie/ccd/ccdlistener.cpp b/roxie/ccd/ccdlistener.cpp index d659928df9e..f575adf82c9 100644 --- a/roxie/ccd/ccdlistener.cpp +++ b/roxie/ccd/ccdlistener.cpp @@ -128,7 +128,7 @@ class CascadeManager : public CInterface if (traceLevel) { StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); DBGLOG("connectChild connecting to %s", epStr.str()); } Owned sock = ISocket::connect_timeout(ep, 2000); @@ -147,7 +147,7 @@ class CascadeManager : public CInterface { StringBuffer err; err.append("Roxie CascadeManager failed to establish secure connection to "); - ep.getUrlStr(err); + ep.getEndpointHostText(err); err.append(": returned ").append(status); throw makeStringException(ROXIE_TLS_ERROR, err.str()); } @@ -158,7 +158,7 @@ class CascadeManager : public CInterface if (traceLevel) { StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); DBGLOG("connectChild connected to %s", epStr.str()); } } @@ -168,7 +168,7 @@ class CascadeManager : public CInterface connectChild((idx+1) * 2 - 1); connectChild((idx+1) * 2); errors.append("").append(E->errorCode()).append(""); E->errorMessage(errors).append(""); logctx.CTXLOG("Connection failed - %s", errors.str()); @@ -491,7 +491,7 @@ class CascadeManager : public CInterface { StringBuffer myReply; myReply.append("\n"); unsigned savedLength = myReply.length(); try @@ -802,7 +802,7 @@ class RoxieListener : public Thread, implements IHpccProtocolListener, implement if (!allowed) { StringBuffer peerStr; - peer.getIpText(peerStr); + peer.getHostText(peerStr); StringBuffer qText; if (queryText && *queryText) decodeXML(queryText, qText); @@ -1387,7 +1387,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte { unsigned instanceId = getNextInstanceId(); StringBuffer ctxstr; - logctx.setown(new StringContextLogger(ep.getIpText(ctxstr).appendf(":%u{%u}", ep.port, instanceId).str())); + logctx.setown(new StringContextLogger(ep.getHostText(ctxstr).appendf(":%u{%u}", ep.port, instanceId).str())); } return *logctx; } @@ -1454,7 +1454,7 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte if (!global && !isEmptyString(logctx->queryGlobalId())) //globalId wins return; StringBuffer s; - ep.getIpText(s).appendf(":%u{%s}", ep.port, uid.str()); //keep no matter what for existing log parsers + ep.getHostText(s).appendf(":%u{%s}", ep.port, uid.str()); //keep no matter what for existing log parsers if (global) { s.append('['); @@ -1696,7 +1696,7 @@ class RoxieProtocolMsgSink : implements IHpccNativeProtocolMsgSink, public CInte if (!allowed) { StringBuffer peerStr; - peer.getIpText(peerStr); + peer.getHostText(peerStr); StringBuffer qText; if (queryText && *queryText) decodeXML(queryText, qText); @@ -1875,7 +1875,7 @@ class RoxieProtocolMsgSink : implements IHpccNativeProtocolMsgSink, public CInte else if (strieq(name, "control:queryaclinfo")) { reply.append("\n"); queryAccessInfo(reply); diff --git a/roxie/ccd/ccdmain.cpp b/roxie/ccd/ccdmain.cpp index adacb59878f..94e66e84cb1 100644 --- a/roxie/ccd/ccdmain.cpp +++ b/roxie/ccd/ccdmain.cpp @@ -1353,7 +1353,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) if (traceLevel) { StringBuffer s; - DBGLOG("My node ip=%s", myIP.getIpText(s).str()); + DBGLOG("My node ip=%s", myIP.getHostText(s).str()); } if (topology->getPropBool("@server", true)) { @@ -1451,7 +1451,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) else { Owned protocolPlugin = loadHpccProtocolPlugin(protocolCtx, NULL); - Owned roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(myNode.getIpAddress(), 0, 1, false), 0, 0, NULL); + Owned roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(myNode.getIpAddress(), 0, 1, false), 0, 0, nullptr, nullptr, nullptr, nullptr, nullptr); try { const char *format = topology->queryProp("@format"); @@ -1502,7 +1502,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) { roxiePort = port; if (roxieFarm.getPropBool("@tls")) - roxiePortTlsClientConfig = createTlsClientSecretInfo(roxieFarm.queryProp("@issuer"), !roxieFarm.getPropBool("@public"), roxieFarm.getPropBool("@selfSigned")); + roxiePortTlsClientConfig = createIssuerTlsClientConfig(roxieFarm.queryProp("@issuer"), roxieFarm.getPropBool("@selfSigned")); debugEndpoint.set(roxiePort, ip); } bool suspended = roxieFarm.getPropBool("@suspended", false); @@ -1514,40 +1514,49 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) StringBuffer certFileName; StringBuffer keyFileName; StringBuffer passPhraseStr; + Owned tlsConfig; if (serviceTLS) { protocol = "ssl"; #ifdef _USE_OPENSSL - #ifdef _CONTAINERIZED - const char *certIssuer = roxieFarm.getPropBool("@public", true) ? "public" : "local"; - certFileName.setf("/opt/HPCCSystems/secrets/certificates/%s/tls.crt", certIssuer); - keyFileName.setf("/opt/HPCCSystems/secrets/certificates/%s/tls.key", certIssuer); - #else - const char *passPhrase = roxieFarm.queryProp("@passphrase"); - if (!isEmptyString(passPhrase)) - decrypt(passPhraseStr, passPhrase); - - const char *certFile = roxieFarm.queryProp("@certificateFileName"); - if (!certFile) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port); - if (isAbsolutePath(certFile)) - certFileName.append(certFile); - else - certFileName.append(codeDirectory.str()).append(certFile); - - const char *keyFile = roxieFarm.queryProp("@privateKeyFileName"); - if (!keyFile) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port); - if (isAbsolutePath(keyFile)) - keyFileName.append(keyFile); + if (isContainerized()) + { + const char *certIssuer = roxieFarm.queryProp("@issuer"); + if (isEmptyString(certIssuer)) + certIssuer = roxieFarm.getPropBool("@public", true) ? "public" : "local"; + tlsConfig.setown(getIssuerTlsServerConfigWithTrustedPeers(certIssuer, roxieFarm.queryProp("trusted_peers"))); + if (!tlsConfig) + throw MakeStringException(ROXIE_FILE_ERROR, "TLS secret for issuer %s not found", certIssuer); + DBGLOG("Roxie service, port(%d) TLS issuer (%s)", port, certIssuer); + } else - keyFileName.append(codeDirectory.str()).append(keyFile); - #endif - if (!checkFileExists(certFileName.str())) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str()); - - if (!checkFileExists(keyFileName.str())) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str()); + { + const char *passPhrase = roxieFarm.queryProp("@passphrase"); + if (!isEmptyString(passPhrase)) + decrypt(passPhraseStr, passPhrase); + + const char *certFile = roxieFarm.queryProp("@certificateFileName"); + if (!certFile) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port); + if (isAbsolutePath(certFile)) + certFileName.append(certFile); + else + certFileName.append(codeDirectory.str()).append(certFile); + + const char *keyFile = roxieFarm.queryProp("@privateKeyFileName"); + if (!keyFile) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port); + if (isAbsolutePath(keyFile)) + keyFileName.append(keyFile); + else + keyFileName.append(codeDirectory.str()).append(keyFile); + + if (!checkFileExists(certFileName.str())) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str()); + + if (!checkFileExists(keyFileName.str())) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str()); + } #else OWARNLOG("Skipping Roxie SSL Farm Listener on port %d : OpenSSL disabled in build", port); @@ -1558,7 +1567,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) const char *config = roxieFarm.queryProp("@config"); // NB: leaks - until we fix bug in ensureProtocolPlugin() whereby some paths return a linked object and others do not IHpccProtocolPlugin *protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname); - roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, certFileName.str(), keyFileName.str(), passPhraseStr.str())); + roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, tlsConfig, certFileName, keyFileName, passPhraseStr)); } else roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended)); diff --git a/roxie/ccd/ccdprotocol.cpp b/roxie/ccd/ccdprotocol.cpp index f0586bbf3eb..66ec3ca8ff8 100644 --- a/roxie/ccd/ccdprotocol.cpp +++ b/roxie/ccd/ccdprotocol.cpp @@ -29,7 +29,7 @@ //================================================================================================================================ -IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile, const char *keyFile, const char *passPhrase); +IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase); class CHpccProtocolPlugin : implements IHpccProtocolPlugin, public CInterface { @@ -60,9 +60,9 @@ class CHpccProtocolPlugin : implements IHpccProtocolPlugin, public CInterface maxHttpConnectionRequests = ctx.ctxGetPropInt("@maxHttpConnectionRequests", 0); maxHttpKeepAliveWait = ctx.ctxGetPropInt("@maxHttpKeepAliveWait", 5000); // In milliseconds } - IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr) + IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase) { - return createProtocolListener(protocol, sink, port, listenQueue, certFile, keyFile, passPhrase); + return createProtocolListener(protocol, sink, port, listenQueue, tlsConfig, certFile, keyFile, passPhrase); } public: StringArray targetNames; @@ -231,7 +231,7 @@ class ProtocolSocketListener : public ProtocolListener bool isSSL = false; public: - ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue, const char *_protocol, const char *_certFile, const char *_keyFile, const char *_passPhrase) + ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue, const char *_protocol, const IPropertyTree *_tlsConfig, const char *_certFile, const char *_keyFile, const char *_passPhrase) : ProtocolListener(_sink) { port = _port; @@ -242,9 +242,15 @@ class ProtocolSocketListener : public ProtocolListener keyFile.set(_keyFile); passPhrase.set(_passPhrase); isSSL = streq(protocol.str(), "ssl"); + #ifdef _USE_OPENSSL if (isSSL) - secureContext.setown(createSecureSocketContextEx(certFile.get(), keyFile.get(), passPhrase.get(), ServerSocket)); + { + if (_tlsConfig) + secureContext.setown(createSecureSocketContextEx2(_tlsConfig, ServerSocket)); + else + secureContext.setown(createSecureSocketContextEx(certFile.get(), keyFile.get(), passPhrase.get(), ServerSocket)); + } #endif } @@ -1820,7 +1826,7 @@ class RoxieSocketWorker : public ProtocolQueryWorker Owned queryPT; StringBuffer sanitizedText; StringBuffer peerStr; - peer.getIpText(peerStr); + peer.getHostText(peerStr); const char *uid = "-"; StringAttr queryName; @@ -2222,11 +2228,16 @@ void ProtocolSocketListener::runOnce(const char *query) p->runOnce(query); } -IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr) +IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase) { if (traceLevel) - DBGLOG("Creating Roxie socket listener, protocol %s, pool size %d, listen queue %d%s", protocol, sink->getPoolSize(), listenQueue, sink->getIsSuspended() ? " SUSPENDED":""); - return new ProtocolSocketListener(sink, port, listenQueue, protocol, certFile, keyFile, passPhrase); + { + const char *certIssuer = "none"; + if (tlsConfig && tlsConfig->hasProp("@issuer")) + certIssuer = tlsConfig->queryProp("@issuer"); + DBGLOG("Creating Roxie socket listener, protocol %s, issuer=%s, pool size %d, listen queue %d%s", protocol, certIssuer, sink->getPoolSize(), listenQueue, sink->getIsSuspended() ? " SUSPENDED":""); + } + return new ProtocolSocketListener(sink, port, listenQueue, protocol, tlsConfig, certFile, keyFile, passPhrase); } extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *_limiterFactory) diff --git a/roxie/ccd/ccdqueue.cpp b/roxie/ccd/ccdqueue.cpp index 27fb6e5eabe..97945643159 100644 --- a/roxie/ccd/ccdqueue.cpp +++ b/roxie/ccd/ccdqueue.cpp @@ -165,7 +165,7 @@ StringBuffer &RoxiePacketHeader::toString(StringBuffer &ret) const default: ret.append("???"); break; } ret.appendf(" queryHash=%" I64F "x ch=%u seq=%d cont=%d server=", queryHash, channel, overflowSequence, continueSequence); - serverIP.getIpText(ret); + serverIP.getHostText(ret); if (retries) { if (retries==QUERY_ABORTED) @@ -269,7 +269,7 @@ void joinMulticastChannel(unsigned channel) getChannelIp(multicastIp, channel); SocketEndpoint ep(ccdMulticastPort, multicastIp); StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); if (!multicastSocket->join_multicast_group(ep)) throw MakeStringException(ROXIE_MULTICAST_ERROR, "Failed to join multicast channel %d (%s)", channel, epStr.str()); if (traceLevel) @@ -908,7 +908,7 @@ void AgentContextLogger::set(ISerializedRoxieQueryPacket *packet) s.append("|"); } channel = header.channel; - ip.getIpText(s); + ip.getHostText(s); s.append(':').append(channel); StringContextLogger::set(s.str()); if (intercept || mergeAgentStatistics) diff --git a/roxie/ccd/hpccprotocol.hpp b/roxie/ccd/hpccprotocol.hpp index cd09e589556..60cdb842bc7 100644 --- a/roxie/ccd/hpccprotocol.hpp +++ b/roxie/ccd/hpccprotocol.hpp @@ -135,7 +135,7 @@ interface IActiveQueryLimiterFactory : extends IInterface interface IHpccProtocolPlugin : extends IInterface { - virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr)=0; + virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase)=0; }; extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *limiterFactory); diff --git a/roxie/roxiepipe/roxiepipe.cpp b/roxie/roxiepipe/roxiepipe.cpp index 632edde60b9..1349d9710c1 100644 --- a/roxie/roxiepipe/roxiepipe.cpp +++ b/roxie/roxiepipe/roxiepipe.cpp @@ -279,7 +279,7 @@ class RoxieThread : public Thread int code = ep->getPropInt("./Code", 0); SocketEndpoint peerEp; StringBuffer peerStr; - OERRLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getUrlStr(peerStr).str()); + OERRLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getEndpointHostText(peerStr).str()); OERRLOG("Roxie exception: %s", body.str()); throw new ReceivedRoxieException(code, body.str()); } @@ -305,7 +305,7 @@ class RoxieThread : public Thread delete x; SocketEndpoint peerEp; StringBuffer peerStr; - OERRLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getUrlStr(peerStr).str()); + OERRLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getEndpointHostText(peerStr).str()); OERRLOG("Roxie exception: %s", xml.str()); int code = 0; try @@ -354,7 +354,7 @@ class RoxieThread : public Thread SocketEndpoint peerEp; StringBuffer peerStr; - PROGLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getUrlStr(peerStr).str()); + PROGLOG("Connected to %s", roxieSock->querySocket()->getPeerEndpoint(peerEp).getEndpointHostText(peerStr).str()); sendQuery(); diff --git a/roxie/udplib/udpaeron.cpp b/roxie/udplib/udpaeron.cpp index 3215a980fcb..b86c773dce7 100644 --- a/roxie/udplib/udpaeron.cpp +++ b/roxie/udplib/udpaeron.cpp @@ -285,7 +285,7 @@ class CRoxieAeronReceiveManager : public CInterfaceOf std::shared_ptr addSubscription(const SocketEndpoint &myEndpoint, int queue) { StringBuffer channel("aeron:udp?endpoint="); - myEndpoint.getUrlStr(channel); + myEndpoint.getEndpointHostText(channel); std::int64_t id = aeron->addSubscription(channel.str(), queue); std::shared_ptr subscription = aeron->findSubscription(id); while (!subscription) @@ -310,7 +310,7 @@ class UdpAeronReceiverEntry : public IUdpReceiverEntry : dest(_ip), aeron(_aeron), numQueues(_numQueues) { StringBuffer channel("aeron:udp?endpoint="); - dest.getIpText(channel); + dest.getHostText(channel); channel.append(':').append(_dataPort); for (unsigned queue = 0; queue < numQueues; queue++) { @@ -358,7 +358,7 @@ class UdpAeronReceiverEntry : public IUdpReceiverEntry continue; } StringBuffer target; - dest.getIpText(target); + dest.getHostText(target); if (aeron::NOT_CONNECTED == result) throw makeStringExceptionV(ROXIE_PUBLICATION_NOT_CONNECTED, "AeronSender: Offer failed because publisher is not connected to subscriber %s", target.str()); else if (aeron::PUBLICATION_CLOSED == result) diff --git a/roxie/udplib/udplib.hpp b/roxie/udplib/udplib.hpp index c832c506540..3d3061227df 100644 --- a/roxie/udplib/udplib.hpp +++ b/roxie/udplib/udplib.hpp @@ -75,7 +75,7 @@ class UDPLIB_API ServerIdentifier { IpAddress serverIp; serverIp.setIP4(netAddress); - return serverIp.getIpText(s); + return serverIp.getHostText(s); } bool isMe() const; }; diff --git a/roxie/udplib/udpsha.cpp b/roxie/udplib/udpsha.cpp index 45d5ac6cb54..18db04b2a56 100644 --- a/roxie/udplib/udpsha.cpp +++ b/roxie/udplib/udpsha.cpp @@ -905,7 +905,7 @@ unsigned CSimulatedQueueWriteSocket::writeDelayed(unsigned now) free((void *) packets.front()); if (jitteredBuff) free((void *) jitteredBuff); - DBGLOG("Write to disconnected socket %s", destEp.getUrlStr(s).str()); + DBGLOG("Write to disconnected socket %s", destEp.getEndpointHostText(s).str()); } dueTimes.pop(); packets.pop(); @@ -932,7 +932,7 @@ size32_t CSimulatedQueueWriteSocket::write(void const* buf, size32_t size) else { StringBuffer s; - DBGLOG("Write to disconnected socket %s", destEp.getUrlStr(s).str()); + DBGLOG("Write to disconnected socket %s", destEp.getEndpointHostText(s).str()); } } return size; @@ -944,7 +944,7 @@ CriticalSection CSimulatedQueueReadSocket::allReadersCrit; CSimulatedQueueReadSocket::CSimulatedQueueReadSocket(const SocketEndpoint &_me) : me(_me) { StringBuffer s; - DBGLOG("Creating fake socket %s", me.getUrlStr(s).str()); + DBGLOG("Creating fake socket %s", me.getEndpointHostText(s).str()); CriticalBlock b(allReadersCrit); allReaders[me] = this; } @@ -952,7 +952,7 @@ CSimulatedQueueReadSocket::CSimulatedQueueReadSocket(const SocketEndpoint &_me) CSimulatedQueueReadSocket::~CSimulatedQueueReadSocket() { StringBuffer s; - DBGLOG("Closing fake socket %s", me.getUrlStr(s).str()); + DBGLOG("Closing fake socket %s", me.getEndpointHostText(s).str()); CriticalBlock b(allReadersCrit); allReaders.erase(me); } @@ -981,7 +981,7 @@ void CSimulatedQueueReadSocket::writeSimulatedPacket(void const* buf, size32_t s packets.push(memcpy(malloc(size), buf, size)); used += size; } -// StringBuffer s; DBGLOG("Signalling available data on %s", me.getUrlStr(s).str()); +// StringBuffer s; DBGLOG("Signalling available data on %s", me.getEndpointHostText(s).str()); avail.signal(); } @@ -999,7 +999,7 @@ void CSimulatedQueueReadSocket::writeOwnSimulatedPacket(void const* buf, size32_ packets.push(buf); used += size; } -// StringBuffer s; DBGLOG("Signalling available data on %s", me.getUrlStr(s).str()); +// StringBuffer s; DBGLOG("Signalling available data on %s", me.getEndpointHostText(s).str()); avail.signal(); } diff --git a/roxie/udplib/udptopo.cpp b/roxie/udplib/udptopo.cpp index ef16bb9e372..bccd20f4f7a 100644 --- a/roxie/udplib/udptopo.cpp +++ b/roxie/udplib/udptopo.cpp @@ -229,7 +229,7 @@ CTopologyServer::CTopologyServer(const char *topologyInfo, const ITopologyServer if (!isActive(instance) || (isActive(oldInstance) && oldInstance != instance)) { StringBuffer s; - DBGLOG("Deleting pending data for server %s which has terminated or restarted", ep.getUrlStr(s).str()); + DBGLOG("Deleting pending data for server %s which has terminated or restarted", ep.getEndpointHostText(s).str()); ROQ->abortPendingData(ep); } if (isActive(instance)) @@ -545,7 +545,7 @@ void TopologyManager::_setRoles(const std::vector &myRoles, b default: throwUnexpected(); } topoBuf.append(role.channel).append('|'); - role.ep.getUrlStr(topoBuf); + getRemoteAccessibleHostText(topoBuf, role.ep); topoBuf.append('|').append(role.replicationLevel); topoBuf.append('\t').append((__uint64) myInstance); topoBuf.append('\n'); diff --git a/roxie/udplib/udptrr.cpp b/roxie/udplib/udptrr.cpp index fa05ca1131d..6e8fe370739 100644 --- a/roxie/udplib/udptrr.cpp +++ b/roxie/udplib/udptrr.cpp @@ -646,7 +646,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceLevel || udpTraceFlow) { StringBuffer s; - DBGLOG("UdpFlow: Duplicate requestToSend %" SEQF "u from node %s", _flowSeq, dest.getIpText(s).str()); + DBGLOG("UdpFlow: Duplicate requestToSend %" SEQF "u from node %s", _flowSeq, dest.getHostText(s).str()); } } @@ -670,14 +670,14 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceLevel > 3 || udpTraceFlow) { StringBuffer ipStr; - DBGLOG("UdpReceiver: sending request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getIpText(ipStr).str()); + DBGLOG("UdpReceiver: sending request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getHostText(ipStr).str()); } #ifdef TEST_DROPPED_PACKETS flowPacketsSent[msg.cmd]++; if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd]==0) { StringBuffer ipStr; - DBGLOG("UdpReceiver: deliberately dropping request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getIpText(ipStr).str()); + DBGLOG("UdpReceiver: deliberately dropping request_received msg seq %" SEQF "u to node=%s", _flowSeq, dest.getHostText(ipStr).str()); } else #endif @@ -688,7 +688,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface catch(IException *e) { StringBuffer d, s; - DBGLOG("UdpReceiver: acknowledgeRequest failed node=%s %s", dest.getIpText(d).str(), e->errorMessage(s).str()); + DBGLOG("UdpReceiver: acknowledgeRequest failed node=%s %s", dest.getHostText(d).str(), e->errorMessage(s).str()); e->Release(); } } @@ -710,14 +710,14 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceLevel > 3 || udpTraceFlow) { StringBuffer ipStr; - DBGLOG("UdpReceiver: sending ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getIpText(ipStr).str()); + DBGLOG("UdpReceiver: sending ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getHostText(ipStr).str()); } #ifdef TEST_DROPPED_PACKETS flowPacketsSent[msg.cmd]++; if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd]==0) { StringBuffer ipStr; - DBGLOG("UdpReceiver: deliberately dropping ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getIpText(ipStr).str()); + DBGLOG("UdpReceiver: deliberately dropping ok_to_send %u msg seq %" SEQF "u to node=%s", maxTransfer, flowSeq, dest.getHostText(ipStr).str()); } else #endif @@ -727,7 +727,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface catch(IException *e) { StringBuffer d, s; - DBGLOG("UdpReceiver: requestToSend failed node=%s %s", dest.getIpText(d).str(), e->errorMessage(s).str()); + DBGLOG("UdpReceiver: requestToSend failed node=%s %s", dest.getHostText(d).str(), e->errorMessage(s).str()); e->Release(); } } @@ -1057,7 +1057,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceLevel || udpTraceFlow || udpTraceTimeouts) { StringBuffer s; - DBGLOG("permit %" SEQF "u to node %s (%u packets) timed out after %u ms, rescheduling", sender->flowSeq, sender->dest.getIpText(s).str(), sender->getTotalReserved(), elapsed); + DBGLOG("permit %" SEQF "u to node %s (%u packets) timed out after %u ms, rescheduling", sender->flowSeq, sender->dest.getHostText(s).str(), sender->getTotalReserved(), elapsed); } SendPermit *next = finger->next; @@ -1069,7 +1069,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceLevel || udpTraceFlow || udpTraceTimeouts) { StringBuffer s; - DBGLOG("permit to send %" SEQF "u to node %s timed out %u times - abandoning", sender->flowSeq, sender->dest.getIpText(s).str(), sender->timeouts); + DBGLOG("permit to send %" SEQF "u to node %s timed out %u times - abandoning", sender->flowSeq, sender->dest.getHostText(s).str(), sender->timeouts); } //Currently this is benign. If the sender really is alive it will send another request. @@ -1158,7 +1158,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceFlow) { StringBuffer s; - DBGLOG("Sender %s can't be given permission to send yet as all permits active", finger->dest.getIpText(s).str()); + DBGLOG("Sender %s can't be given permission to send yet as all permits active", finger->dest.getHostText(s).str()); } } } @@ -1168,7 +1168,7 @@ class CReceiveManager : implements IReceiveManager, public CInterface if (udpTraceFlow) { StringBuffer s; - DBGLOG("Sender %s can't be given permission to send yet as resend buffer full", finger->dest.getIpText(s).str()); + DBGLOG("Sender %s can't be given permission to send yet as resend buffer full", finger->dest.getHostText(s).str()); } } } diff --git a/roxie/udplib/udptrs.cpp b/roxie/udplib/udptrs.cpp index 2ead7e043f0..b75da03e201 100644 --- a/roxie/udplib/udptrs.cpp +++ b/roxie/udplib/udptrs.cpp @@ -217,14 +217,14 @@ class UdpReceiverEntry : public IUdpReceiverEntry { StringBuffer s, s2; DBGLOG("UdpSender[%s]: sending flowType::%s msg %" SEQF "u flowSeq %" SEQF "u size=%u to node=%s %s", - msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, msg.packets, ip.getIpText(s).str(), sendWithData ? "" : ""); + msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, msg.packets, ip.getHostText(s).str(), sendWithData ? "" : ""); } #ifdef TEST_DROPPED_PACKETS flowPacketsSent[msg.cmd]++; if (udpDropFlowPackets[msg.cmd] && flowPacketsSent[msg.cmd]%udpDropFlowPackets[msg.cmd] == 0) { StringBuffer s, s2; - DBGLOG("UdpSender[%s]: deliberately dropping flowType::%s msg %" SEQF "u flowSeq %" SEQF "u to node=%s", msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, ip.getIpText(s).str()); + DBGLOG("UdpSender[%s]: deliberately dropping flowType::%s msg %" SEQF "u flowSeq %" SEQF "u to node=%s", msg.sourceNode.getTraceText(s2).str(), flowType::name(msg.cmd), msg.sendSeq, msg.flowSeq, ip.getHostText(s).str()); } else #endif @@ -385,7 +385,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry int timeExpired = msTick()-requestExpiryTime; StringBuffer s; EXCLOG(MCoperatorError,"ERROR: UdpSender: timed out %i times (flow=%u, max=%i, timeout=%u, expiryTime=%u[%u] ack(%u)) waiting ok_to_send msg from node=%s", - timeouts.load(), activeFlowSequence.load(), maxRequestDeadTimeouts, udpFlowAckTimeout, requestExpiryTime.load(), timeExpired, (int)hadAcknowledgement, ip.getIpText(s).str()); + timeouts.load(), activeFlowSequence.load(), maxRequestDeadTimeouts, udpFlowAckTimeout, requestExpiryTime.load(), timeExpired, (int)hadAcknowledgement, ip.getHostText(s).str()); } } @@ -455,7 +455,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry if (permit.destNode.getIpAddress().ipcompare(ip) != 0) { StringBuffer p, s; - DBGLOG("UdpFlow: permit ip %s does not match receiver table ip %s", permit.destNode.getTraceText(p).str(), ip.getIpText(s).str()); + DBGLOG("UdpFlow: permit ip %s does not match receiver table ip %s", permit.destNode.getTraceText(p).str(), ip.getHostText(s).str()); printStackReport(); } #endif @@ -652,7 +652,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry if (udpTraceLevel > 3) { StringBuffer s; - DBGLOG("UdpSender: abort sending queued data to node=%s", ip.getIpText(s).str()); + DBGLOG("UdpSender: abort sending queued data to node=%s", ip.getHostText(s).str()); } timeouts = 0; requestExpiryTime = 0; @@ -753,13 +753,13 @@ class UdpReceiverEntry : public IUdpReceiverEntry catch(IException *e) { StringBuffer error, ipstr; - DBGLOG("UdpSender: udp_connect failed %s %s", ip.getIpText(ipstr).str(), e->errorMessage(error).str()); + DBGLOG("UdpSender: udp_connect failed %s %s", ip.getHostText(ipstr).str(), e->errorMessage(error).str()); throw; } catch(...) { StringBuffer ipstr; - DBGLOG("UdpSender: udp_connect failed %s %s", ip.getIpText(ipstr).str(), "Unknown error"); + DBGLOG("UdpSender: udp_connect failed %s %s", ip.getHostText(ipstr).str(), "Unknown error"); throw; } output_queue = new queue_t[numQueues]; @@ -773,7 +773,7 @@ class UdpReceiverEntry : public IUdpReceiverEntry if (udpTraceLevel > 0) { StringBuffer ipStr, myIpStr; - DBGLOG("UdpSender[%s]: added entry for ip=%s to receivers table - send_flow_port=%d", _myIP.getIpText(myIpStr).str(), ip.getIpText(ipStr).str(), _sendFlowPort); + DBGLOG("UdpSender[%s]: added entry for ip=%s to receivers table - send_flow_port=%d", _myIP.getHostText(myIpStr).str(), ip.getHostText(ipStr).str(), _sendFlowPort); } } if (udpResendLostPackets) @@ -860,7 +860,7 @@ class CSendManager : implements ISendManager, public CInterface if (&receiverInfo != &dest) { StringBuffer s; - DBGLOG("UdpSender[%s]: table entry %s does not find itself", parent.myId, dest.ip.getIpText(s).str()); + DBGLOG("UdpSender[%s]: table entry %s does not find itself", parent.myId, dest.ip.getHostText(s).str()); printStackReport(); } @@ -881,7 +881,7 @@ class CSendManager : implements ISendManager, public CInterface else if (udpTraceFlow && (timeToGo < 0)) { StringBuffer s; - DBGLOG("UdpSender[%s]: entry %s timeout waiting to send with active permit", parent.myId, dest.ip.getIpText(s).str()); + DBGLOG("UdpSender[%s]: entry %s timeout waiting to send with active permit", parent.myId, dest.ip.getHostText(s).str()); } } } @@ -1161,7 +1161,7 @@ class CSendManager : implements ISendManager, public CInterface receiversTable([_numQueues, q_size, server_flow_port, data_port, _encrypted, this](const ServerIdentifier ip) { return new UdpReceiverEntry(ip.getIpAddress(), myIP, _numQueues, q_size, server_flow_port, data_port, _encrypted);}), encrypted(_encrypted) { - myId = myIP.getIpText(myIdStr).str(); + myId = myIP.getHostText(myIdStr).str(); #ifndef _WIN32 if (udpAdjustThreadPriorities) setpriority(PRIO_PROCESS, 0, -3); diff --git a/roxie/udplib/uttest.cpp b/roxie/udplib/uttest.cpp index da2a572ab06..a5c19254f5e 100644 --- a/roxie/udplib/uttest.cpp +++ b/roxie/udplib/uttest.cpp @@ -735,7 +735,7 @@ int main(int argc, char * argv[] ) ipstr.append(startrange - ip, ip).append(firstnum).append(endptr); const IpAddress nodeIP(ipstr); allNodes.append(nodeIP); - nodeIP.getIpText(ipstr.clear()); + nodeIP.getHostText(ipstr.clear()); printf("Added node %s\n", ipstr.str()); firstnum++; } diff --git a/system/hrpc/hrpcmp.cpp b/system/hrpc/hrpcmp.cpp index 5ad7693a994..a70193b394c 100644 --- a/system/hrpc/hrpcmp.cpp +++ b/system/hrpc/hrpcmp.cpp @@ -481,7 +481,7 @@ class HRPCmptransport: implements IHRPCtransport, public CInterface return false; } StringBuffer str; - ep.getUrlStr(str).appendf("{%d}",(int)tag); // we'll use proper tag name when available + ep.getEndpointHostText(str).appendf("{%d}",(int)tag); // we'll use proper tag name when available size32_t l = str.length(); if (l>=namemax) l = namemax-1; diff --git a/system/hrpc/hrpcsock.cpp b/system/hrpc/hrpcsock.cpp index 5111c45b167..cbbbc37c0c6 100644 --- a/system/hrpc/hrpcsock.cpp +++ b/system/hrpc/hrpcsock.cpp @@ -177,7 +177,7 @@ class HRPCsockettransport: implements IHRPCtransport, public CInterface } try { StringBuffer ipname; - endpoint.getIpText(ipname); + endpoint.getHostText(ipname); serversock = ISocket::create_ip(endpoint.port,ipname.str(),qsize); } catch (IJSOCK_Exception *e) { @@ -314,7 +314,7 @@ class HRPCsockettransport: implements IHRPCtransport, public CInterface { free(hostname); StringBuffer ipname; - endpoint.getIpText(ipname); + endpoint.getHostText(ipname); hostname = strdup(ipname.str()); hostport = endpoint.port; } diff --git a/system/hrpc/hrpcutil.cpp b/system/hrpc/hrpcutil.cpp index 278c33d4eef..fc9d4e9609e 100644 --- a/system/hrpc/hrpcutil.cpp +++ b/system/hrpc/hrpcutil.cpp @@ -140,7 +140,7 @@ bool FastMultipleConnect(unsigned n,HRPCmodule **modules,bool *done,int timeout) } else { StringBuffer epstr; - eps.item(i).getUrlStr(epstr); + eps.item(i).getEndpointHostText(epstr); //ERRLOG("Failed to connect to %s",epstr.str()); ret = false; } diff --git a/system/jlib/jcontainerized.cpp b/system/jlib/jcontainerized.cpp index c4172e4828a..093e647834b 100644 --- a/system/jlib/jcontainerized.cpp +++ b/system/jlib/jcontainerized.cpp @@ -19,7 +19,6 @@ namespace k8s { -#ifdef _CONTAINERIZED static StringBuffer myPodName; const char *queryMyPodName() @@ -63,13 +62,45 @@ void deleteResource(const char *componentName, const char *resourceType, const c remove(k8sResourcesFilename); } +bool checkExitCodes(StringBuffer &output, const char *podStatuses) +{ + const char *startOfPodStatus = podStatuses; + while (*startOfPodStatus) + { + const char *endOfPodStatus = strchr(startOfPodStatus, '|'); + StringBuffer podStatus; + if (endOfPodStatus) + podStatus.append((size_t)(endOfPodStatus-startOfPodStatus), startOfPodStatus); + else + podStatus.append(startOfPodStatus); + StringArray fields; + fields.appendList(podStatus, ","); + if (3 == fields.length()) // should be 3 fields {,<"initContainer"|"container">,} + { + const char *exitCodeStr = fields.item(0); + if (strlen(exitCodeStr)) + { + unsigned exitCode = atoi(exitCodeStr); + if (exitCode) // non-zero = failure + { + output.appendf(" %s '%s' failed with exitCode = %u", fields.item(1), fields.item(2), exitCode); + return true; + } + } + } + if (!endOfPodStatus) + break; + startOfPodStatus = endOfPodStatus+1; + } + return false; +} + void waitJob(const char *componentName, const char *resourceType, const char *job, unsigned pendingTimeoutSecs, KeepJobs keepJob) { VStringBuffer jobName("%s-%s-%s", componentName, resourceType, job); jobName.toLowerCase(); VStringBuffer waitJob("kubectl get jobs %s -o jsonpath={.status.active}", jobName.str()); VStringBuffer getScheduleStatus("kubectl get pods --selector=job-name=%s --output=jsonpath={.items[*].status.conditions[?(@.type=='PodScheduled')].status}", jobName.str()); - VStringBuffer checkJobExitCode("kubectl get pods --selector=job-name=%s --output=jsonpath={.items[*].status.containerStatuses[?(@.name==\"%s\")].state.terminated.exitCode}", jobName.str(), jobName.str()); unsigned delay = 100; unsigned start = msTick(); @@ -82,14 +113,30 @@ void waitJob(const char *componentName, const char *resourceType, const char *jo { StringBuffer output; runKubectlCommand(componentName, waitJob, nullptr, &output); - if (!streq(output, "1")) // status.active value + if ((0 == output.length()) || streq(output, "0")) // status.active value { // Job is no longer active - we can terminate DBGLOG("kubectl jobs output: %s", output.str()); - runKubectlCommand(componentName, checkJobExitCode, nullptr, &output.clear()); - if (output.length() && !streq(output, "0")) // state.terminated.exitCode - throw makeStringExceptionV(0, "Failed to run %s: pod exited with error: %s", jobName.str(), output.str()); - break; + VStringBuffer checkJobExitStatus("kubectl get jobs %s '-o=jsonpath={range .status.conditions[*]}{.type}: {.status} - {.message}|{end}'", jobName.str()); + runKubectlCommand(componentName, checkJobExitStatus, nullptr, &output.clear()); + if (strstr(output.str(), "Failed: ")) + { + VStringBuffer errMsg("Job %s failed [%s].", jobName.str(), output.str()); + VStringBuffer checkInitContainerExitCodes("kubectl get pods --selector=job-name=%s '-o=jsonpath={range .items[*].status.initContainerStatuses[*]}{.state.terminated.exitCode},{\"initContainer\"},{.name}{\"|\"}{end}'", jobName.str()); + runKubectlCommand(componentName, checkInitContainerExitCodes, nullptr, &output.clear()); + DBGLOG("checkInitContainerExitCodes - output = %s", output.str()); + if (!checkExitCodes(errMsg, output)) + { + // no init container failures, check regular containers + VStringBuffer checkContainerExitCodes("kubectl get pods --selector=job-name=%s '-o=jsonpath={range .items[*].status.containerStatuses[*]}{.state.terminated.exitCode},{\"container\"},{.name}{\"|\"}{end}'", jobName.str()); + runKubectlCommand(componentName, checkContainerExitCodes, nullptr, &output.clear()); + DBGLOG("checkContainerExitCodes - output = %s", output.str()); + checkExitCodes(errMsg, output); + } + throw makeStringException(0, errMsg); + } + else // assume success, either .status.conditions type of "Complete" or "Succeeded" + break; } runKubectlCommand(nullptr, getScheduleStatus, nullptr, &output.clear()); @@ -261,51 +308,8 @@ MODULE_INIT(INIT_PRIORITY_STANDARD) } MODULE_EXIT() { - removeConfigUpdateHook(podInfoInitCBId); -} - -#else - -const char *queryMyPodName() -{ - throwUnexpected(); -} - -KeepJobs translateKeepJobs(const char *keepJobs) -{ - throwUnexpected(); -} - -bool isActiveService(const char *serviceName) -{ - throwUnexpected(); -} - -void deleteResource(const char *componentName, const char *job, const char *resource) -{ - throwUnexpected(); -} - -void waitJob(const char *componentName, const char *resourceType, const char *job, unsigned pendingTimeoutSecs, KeepJobs keepJob) -{ - throwUnexpected(); -} - -bool applyYaml(const char *componentName, const char *wuid, const char *job, const char *resourceType, const std::list> &extraParams, bool optional, bool autoCleanup) -{ - throwUnexpected(); -} - -void runJob(const char *componentName, const char *wuid, const char *job, const std::list> &extraParams) -{ - throwUnexpected(); -} - -std::vector> getPodNodes(const char *selector) -{ - throwUnexpected(); + if (isContainerized()) + removeConfigUpdateHook(podInfoInitCBId); } -#endif // _CONTAINERIZED - } // end of k8s namespace diff --git a/system/jlib/jexcept.cpp b/system/jlib/jexcept.cpp index a716380ec1b..035d6243990 100644 --- a/system/jlib/jexcept.cpp +++ b/system/jlib/jexcept.cpp @@ -1313,7 +1313,7 @@ NO_SANITIZE("alignment") void excsighandler(int signum, siginfo_t *info, void *e StringBuffer networkIp; PROGLOG("================================================"); - PROGLOG("Program: %s:%s", queryHostIP().getIpText(networkIp).str(),queryCurrentProcessPath()); + PROGLOG("Program: %s:%s", queryHostIP().getHostText(networkIp).str(),queryCurrentProcessPath()); PROGLOG("Signal: %d %s",signum,strsignal(signum)); PROGLOG("Fault IP: " I64X "", ip); PROGLOG("Accessing: " I64X "", (unsigned __int64) info->si_addr); @@ -1371,7 +1371,7 @@ NO_SANITIZE("alignment") void excsighandler(int signum, siginfo_t *info, void *e StringBuffer networkIp; PROGLOG("================================================"); - PROGLOG("Program: %s:%s", queryHostIP().getIpText(networkIp).str(),queryCurrentProcessPath()); + PROGLOG("Program: %s:%s", queryHostIP().getHostText(networkIp).str(),queryCurrentProcessPath()); PROGLOG("Signal: %d %s",signum,strsignal(signum)); PROGLOG("Fault IP: %08X", ip); PROGLOG("Accessing: %08X", (unsigned) info->si_addr); diff --git a/system/jlib/jfile.cpp b/system/jlib/jfile.cpp index 368034e731b..fa928407783 100644 --- a/system/jlib/jfile.cpp +++ b/system/jlib/jfile.cpp @@ -3396,7 +3396,7 @@ StringBuffer &createUNCFilename(const char * filename, StringBuffer &UNC, bool u if (useHostNames) UNC.append(GetCachedHostName()); else - queryHostIP().getIpText(UNC); + queryHostIP().getHostText(UNC); UNC.append("\\").append((char)tolower(buf[0])).append(getShareChar()).append(buf+2); } else @@ -3414,7 +3414,7 @@ StringBuffer &createUNCFilename(const char * filename, StringBuffer &UNC, bool u if (useHostNames) UNC.append(GetCachedHostName()); else - queryHostIP().getIpText(UNC); + queryHostIP().getHostText(UNC); if (*filename != '/') { @@ -4794,7 +4794,7 @@ StringBuffer & RemoteFilename::getRemotePath(StringBuffer & out) const char c=getPathSeparator(); out.append(c).append(c); - ep.getUrlStr(out); + ep.getEndpointHostText(out); const char *fn; StringBuffer loc; if (sharehead.length()) @@ -4964,7 +4964,7 @@ void RemoteFilename::setPath(const SocketEndpoint & _ep, const char * _filename) tailpath.clear(); #ifdef _TRACERFN StringBuffer eps; - PROGLOG("setPath (%s,%s) -> '%s' '%s' '%s'",ep.getUrlStr(eps).str(),_filename?_filename:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL"); + PROGLOG("setPath (%s,%s) -> '%s' '%s' '%s'",ep.getEndpointHostText(eps).str(),_filename?_filename:"NULL",sharehead.get()?sharehead.get():"NULL",localhead.get()?localhead.get():"NULL",tailpath.get()?tailpath.get():"NULL"); #endif } @@ -5962,7 +5962,7 @@ bool mountDrive(const char *drv,const RemoteFilename &rfn) for (unsigned vtry=0;vtry<2;vtry++) { StringBuffer cmd; cmd.append("mount "); - rfn.queryIP().getIpText(cmd); + rfn.queryIP().getHostText(cmd); cmd.append(':'); rfn.getLocalPath(cmd); cmd.append(' ').append(drv).append(" -t nfs "); diff --git a/system/jlib/jlog.cpp b/system/jlib/jlog.cpp index 8e44015fb6d..6d813f878e5 100644 --- a/system/jlib/jlog.cpp +++ b/system/jlib/jlog.cpp @@ -413,7 +413,7 @@ StringBuffer & LogMsg::toStringPlain(StringBuffer & out, unsigned fields) const } if(fields & MSGFIELD_node) { - sysInfo.queryNode()->getUrlStr(out); + sysInfo.queryNode()->getEndpointHostText(out); out.append(" "); } if(fields & MSGFIELD_job) @@ -494,7 +494,7 @@ StringBuffer & LogMsg::toStringXML(StringBuffer & out, unsigned fields) const if(fields & MSGFIELD_node) { out.append("Node=\""); - sysInfo.queryNode()->getUrlStr(out); + sysInfo.queryNode()->getEndpointHostText(out); out.append("\" "); } #ifdef LOG_MSG_NEWLINE @@ -572,7 +572,7 @@ StringBuffer & LogMsg::toStringJSON(StringBuffer & out, unsigned fields) const if(fields & MSGFIELD_node) { out.append(", \"NODE\": \""); - sysInfo.queryNode()->getUrlStr(out); + sysInfo.queryNode()->getEndpointHostText(out); out.append("\""); } if(fields & MSGFIELD_job) @@ -643,7 +643,7 @@ StringBuffer & LogMsg::toStringTable(StringBuffer & out, unsigned fields) const if(fields & MSGFIELD_node) { size32_t len = out.length(); - sysInfo.queryNode()->getUrlStr(out); + sysInfo.queryNode()->getEndpointHostText(out); out.appendN(20 + len - out.length(), ' '); } if(fields & MSGFIELD_job) @@ -781,7 +781,7 @@ void NodeLogMsgFilter::addToPTree(IPropertyTree * tree) const IPropertyTree * filterTree = createPTree(ipt_caseInsensitive); filterTree->setProp("@type", "node"); StringBuffer buff; - node.getIpText(buff); + node.getHostText(buff); filterTree->setProp("@ip", buff.str()); filterTree->setPropInt("@port", node.port); if(localFlag) filterTree->setPropInt("@local", 1); @@ -793,7 +793,7 @@ void IpLogMsgFilter::addToPTree(IPropertyTree * tree) const IPropertyTree * filterTree = createPTree(ipt_caseInsensitive); filterTree->setProp("@type", "ip"); StringBuffer buff; - ip.getIpText(buff); + ip.getHostText(buff); filterTree->setProp("@ip", buff.str()); if(localFlag) filterTree->setPropInt("@local", 1); tree->addPropTree("filter", filterTree); diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 8a99f467dc7..c5b30c42187 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -92,6 +92,8 @@ MODULE_EXIT() udpKey.clear(); } +static IPropertyTree *getLocalSecret(const char *category, const char * name); + //based on kubernetes secret / key names. Even if some vault backends support additional characters we'll restrict to this subset for now static const char *validSecretNameChrs = ".-"; @@ -240,6 +242,61 @@ extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, splitUrlAuthority(authority, authorityLen, user, password, hostPort, nullptr); } + +static StringBuffer &replaceExtraHostAndPortChars(StringBuffer &s) +{ + size_t l = s.length(); + for (size_t i = 0; i < l; i++) + { + if (s.charAt(i) == '.' || s.charAt(i) == ':') + s.setCharAt(i, '-'); + } + return s; +} + + +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *scheme, const char *userPasswordPair, const char *host, unsigned port, const char *path) +{ + secretName.set("http-connect-"); + //Having the host and port visible will help with manageability wherever the secret is stored + if (scheme && !strnicmp("https", scheme, 5)) + secretName.append("ssl-"); + secretName.append(host); + //port is optionally already part of host + replaceExtraHostAndPortChars(secretName); + if (port) + secretName.append('-').append(port); + //Path and username are both sensitive and shouldn't be accessible in the name, include both in the hash to give us the uniqueness we need + unsigned hashvalue = 0; + if (!isEmptyString(path)) + hashvalue = hashcz((const unsigned char *)path, hashvalue); + if (!isEmptyString(userPasswordPair)) + { + const char *delim = strchr(userPasswordPair, ':'); + //Make unique for a given username, but not the current password. The pw provided could change but what's in the secret (if there is one) wins + if (delim) + hashvalue = hashc((const unsigned char *)userPasswordPair, delim-userPasswordPair, hashvalue); + else + hashvalue = hashcz((const unsigned char *)userPasswordPair, hashvalue); + } + if (hashvalue) + secretName.appendf("-%x", hashvalue); + return secretName; +} + +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *url, const char *inputUsername) +{ + StringBuffer username; + StringBuffer urlPassword; + StringBuffer scheme; + StringBuffer hostPort; + StringBuffer path; + splitUrlIsolateScheme(url, username, urlPassword, scheme, hostPort, path); + if (!isEmptyString(inputUsername)) + username.set(inputUsername); + + return generateDynamicUrlSecretName(secretName, scheme, username, hostPort, 0, path); +} //--------------------------------------------------------------------------------------------------------------------- @@ -272,7 +329,7 @@ extern jlib_decl void setSecretMount(const char * path) secretDirectory.set(path); } -static inline bool checkSecretExpired(unsigned created) +static bool checkSecretExpired(unsigned created) { if (!created) return false; @@ -280,6 +337,11 @@ static inline bool checkSecretExpired(unsigned created) return age > getSecretTimeout(); } +static bool hasCacheExpired(const IPropertyTree * secret) +{ + return checkSecretExpired((unsigned)secret->getPropInt("@created")); +} + enum class VaultAuthType {unknown, k8s, appRole, token}; static void setTimevalMS(timeval &tv, time_t ms) @@ -570,18 +632,16 @@ class CVault IPropertyTree *envelope = tree->queryPropTree(vername); if (!envelope) return false; - if (checkSecretExpired((unsigned) envelope->getPropInt("@created"))) + if (hasCacheExpired(envelope)) { tree->removeTree(envelope); return false; } const char *s = envelope->queryProp(""); + rkind = kind; if (!isEmptyString(s)) - { - rkind = kind; content.append(s); - return true; - } + return true; } return false; } @@ -590,7 +650,8 @@ class CVault VStringBuffer vername("v.%s", isEmptyString(version) ? "latest" : version); Owned envelope = createPTree(vername); envelope->setPropInt("@created", (int) msTick()); - envelope->setProp("", content); + if (!isEmptyString(content)) + envelope->setProp("", content); { CriticalBlock block(vaultCS); IPropertyTree *parent = ensurePTree(cache, secret); @@ -649,6 +710,8 @@ class CVault } else OERRLOG("Error: Vault %s http error (%d) accessing secret %s.%s location %s", name.str(), res.error(), secretCacheKey, version ? version : "", location); + + addCachedSecret("", secretCacheKey, version); //cache misses so we don't keep calling the vault return false; } bool requestSecret(CVaultKind &rkind, StringBuffer &content, const char *secret, const char *version) @@ -800,7 +863,7 @@ IVaultManager *ensureVaultManager() return vaultManager; } -static IPropertyTree *getCachedLocalSecret(const char *category, const char *name) +static IPropertyTree *getCachedLocalSecret(const char *category, const char *name, bool &cachedMiss) { if (isEmptyString(name)) return nullptr; @@ -813,11 +876,16 @@ static IPropertyTree *getCachedLocalSecret(const char *category, const char *nam secret.setown(tree->getPropTree(name)); if (secret) { - if (checkSecretExpired((unsigned) secret->getPropInt("@created"))) + if (hasCacheExpired(secret)) { secretCache->removeProp(name); return nullptr; } + if (secret->hasProp("@miss")) + { + cachedMiss = true; + return nullptr; + } return secret.getClear(); } } @@ -877,12 +945,15 @@ static IPropertyTree *loadLocalSecret(const char *category, const char * name) return tree.getClear(); } -extern jlib_decl IPropertyTree *getLocalSecret(const char *category, const char * name) +static IPropertyTree *getLocalSecret(const char *category, const char * name) { validateCategoryName(category); validateSecretName(name); - Owned tree = getCachedLocalSecret(category, name); + bool skipLocalFetch = false; + Owned tree = getCachedLocalSecret(category, name, skipLocalFetch); + if (skipLocalFetch) + return nullptr; if (tree) return tree.getClear(); return loadLocalSecret(category, name); @@ -908,7 +979,7 @@ static IPropertyTree *createPTreeFromVaultSecret(const char *content, CVaultKind } return tree.getClear(); } -static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version) +static IPropertyTree *getCachedVaultSecret(const char *category, const char *vaultId, const char * name, const char *version, bool &cachedMiss) { CVaultKind kind; StringBuffer json; @@ -923,6 +994,11 @@ static IPropertyTree *getCachedVaultSecret(const char *category, const char *vau if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) return nullptr; } + if (json.isEmpty()) + { + cachedMiss = true; + return nullptr; + } return createPTreeFromVaultSecret(json.str(), kind); } @@ -944,58 +1020,83 @@ static IPropertyTree *requestVaultSecret(const char *category, const char *vault return createPTreeFromVaultSecret(json.str(), kind); } -extern jlib_decl IPropertyTree *getVaultSecret(const char *category, const char *vaultId, const char * name, const char *version) +static IPropertyTree *getVaultSecret(const char *category, const char * name, const char *vaultId, const char *version) { - validateCategoryName(category); - validateSecretName(name); - CVaultKind kind; StringBuffer json; IVaultManager *vaultmgr = ensureVaultManager(); + + bool cachedMiss = false; + if (isEmptyString(vaultId)) { - if (!vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) + if (vaultmgr->getCachedSecretByCategory(category, kind, json, name, version)) + cachedMiss = json.isEmpty(); + else vaultmgr->requestSecretByCategory(category, kind, json, name, version); } else { if (!vaultmgr->getCachedSecretFromVault(category, vaultId, kind, json, name, version)) + cachedMiss = json.isEmpty(); + else vaultmgr->requestSecretFromVault(category, vaultId, kind, json, name, version); } + if (cachedMiss) + return nullptr; return createPTreeFromVaultSecret(json.str(), kind); } -extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name) +IPropertyTree *getSecretTree(const char *category, const char * name, const char * optVaultId, const char * optVersion) { - validateCategoryName(category); - validateSecretName(name); + if (!isEmptyString(optVaultId)) + return getVaultSecret(category, name, optVaultId, optVersion); + + //if we get back a null secret, it might be a cached miss, so don't go to the source if flag gets set + bool skipVaultFetch = false; + bool skipLocalFetch = false; //check for any chached first - Owned secret = getCachedLocalSecret(category, name); + Owned secret = getCachedLocalSecret(category, name, skipLocalFetch); if (!secret) - secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr)); + secret.setown(getCachedVaultSecret(category, nullptr, name, nullptr, skipVaultFetch)); //now check local, then vaults - if (!secret) + if (!secret && !skipLocalFetch) secret.setown(loadLocalSecret(category, name)); - if (!secret) + if (!secret && !skipVaultFetch) secret.setown(requestVaultSecret(category, nullptr, name, nullptr)); return secret.getClear(); } -extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, IPropertyTree *secret, const char * key) +IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) +{ + validateCategoryName(category); + validateSecretName(name); + + return getSecretTree(category, name, optVaultId, optVersion); +} + + +bool getSecretKeyValue(MemoryBuffer & result, const IPropertyTree *secret, const char * key) { validateKeyName(key); + if (!secret) + return false; + IPropertyTree *tree = secret->queryPropTree(key); if (tree) return tree->getPropBin(nullptr, result); return false; } -extern jlib_decl bool getSecretKeyValue(StringBuffer & result, IPropertyTree *secret, const char * key) +bool getSecretKeyValue(StringBuffer & result, const IPropertyTree *secret, const char * key) { validateKeyName(key); + if (!secret) + return false; + IPropertyTree *tree = secret->queryPropTree(key); if (!tree) return false; @@ -1018,8 +1119,6 @@ extern jlib_decl bool getSecretKeyValue(StringBuffer & result, IPropertyTree *se extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required) { - validateKeyName(key); //name and category validated in getSecret - Owned secret = getSecret(category, name); if (required && !secret) throw MakeStringException(-1, "secret %s.%s not found", category, name); @@ -1029,6 +1128,130 @@ extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category return true; } +//--------------------------------------------------------------------------------------------------------------------- + +class CSecret final : public CInterfaceOf +{ +public: + CSecret(const char *_category, const char * _name, const char * _vaultId, const char * _version, const IPropertyTree * _secret) + : category(_category), name(_name), vaultId(_vaultId), version(_version), secret(_secret) + { + updateHash(); + } + + virtual const IPropertyTree * getTree() const; + + virtual bool getKeyValue(MemoryBuffer & result, const char * key) const + { + CriticalBlock block(secretCs); + checkStale(); + return getSecretKeyValue(result, secret, key); + } + virtual bool getKeyValue(StringBuffer & result, const char * key) const + { + CriticalBlock block(secretCs); + checkStale(); + return getSecretKeyValue(result, secret, key); + } + virtual bool isStale() const + { + return secret && hasCacheExpired(secret); + } + virtual unsigned getVersion() const + { + return secretHash; + } + +protected: + void checkStale() const; + void updateHash() const; + +protected: + StringAttr category; + StringAttr name; + StringAttr vaultId; + StringAttr version; + mutable CriticalSection secretCs; + mutable Linked secret; + mutable unsigned secretHash = 0; +}; + + +const IPropertyTree * CSecret::getTree() const +{ + CriticalBlock block(secretCs); + checkStale(); + return LINK(secret); +} + +void CSecret::checkStale() const +{ + if (isStale()) + { + //MORE: This could block or fail - in roxie especially it would be better to return the old value + try + { + secret.setown(getSecretTree(category, name, vaultId, version)); + updateHash(); + } + catch (IException * e) + { + VStringBuffer msg("Failed to update secret %s.%s", category.str(), name.str()); + EXCLOG(e, msg.str()); + e->Release(); + } + } +} + +//This should probably move to jptree.?pp as a generally useful function +static unsigned calculateTreeHash(const IPropertyTree & source, unsigned hashcode) +{ + if (source.isBinary()) + { + MemoryBuffer mb; + source.getPropBin(nullptr, mb); + hashcode = hashc((const byte *)mb.bufferBase(), mb.length(), hashcode); + } + else + { + const char * value = source.queryProp(nullptr); + if (value) + hashcode = hashcz((const byte *)value, hashcode); + } + + Owned aiter = source.getAttributes(); + ForEach(*aiter) + { + hashcode = hashcz((const byte *)aiter->queryName(), hashcode); + hashcode = hashcz((const byte *)aiter->queryValue(), hashcode); + } + + Owned iter = source.getElements("*"); + ForEach(*iter) + { + IPropertyTree & child = iter->query(); + hashcode = hashcz((const byte *)child.queryName(), hashcode); + hashcode = calculateTreeHash(child, hashcode); + } + return hashcode; +} + +void CSecret::updateHash() const +{ + if (secret) + secretHash = calculateTreeHash(*secret.get(), 0x811C9DC5); + else + secretHash = 0; +} + +ISecret * resolveSecret(const char *category, const char * name, const char * optVaultId, const char * optVersion) +{ + Owned resolved = getSecret(category, name, optVaultId, optVersion); + return new CSecret(category, name, optVaultId, optVersion, resolved); +} + +//--------------------------------------------------------------------------------------------------------------------- + void initSecretUdpKey() { if (udpKeyInitialized) @@ -1085,7 +1308,7 @@ jlib_decl bool containsEmbeddedKey(const char *certificate) return false; } -IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert) +IPropertyTree *createIssuerTlsClientConfig(const char *issuer, bool acceptSelfSigned, bool addCACert) { if (isEmptyString(issuer)) return nullptr; @@ -1096,7 +1319,7 @@ IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool a Owned info = createPTree(); - if (mutual) + if (strieq(issuer, "remote")||strieq(issuer, "local")) { filepath.set(secretpath).append("tls.crt"); if (!checkFileExists(filepath)) @@ -1126,7 +1349,7 @@ IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool a return info.getClear(); } -IPropertyTree *queryTlsSecretInfo(const char *name) +IPropertyTree *getIssuerTlsServerConfig(const char *name) { if (isEmptyString(name)) return nullptr; @@ -1134,9 +1357,9 @@ IPropertyTree *queryTlsSecretInfo(const char *name) validateSecretName(name); CriticalBlock block(mtlsInfoCacheCS); - IPropertyTree *info = mtlsInfoCache->queryPropTree(name); + Owned info = mtlsInfoCache->getPropTree(name); if (info) - return info; + return info.getClear(); StringBuffer filepath; StringBuffer secretpath; @@ -1147,7 +1370,8 @@ IPropertyTree *queryTlsSecretInfo(const char *name) if (!checkFileExists(filepath)) return nullptr; - info = mtlsInfoCache->setPropTree(name); + info.set(mtlsInfoCache->setPropTree(name)); + info->setProp("@issuer", name); info->setProp("certificate", filepath.str()); filepath.set(secretpath).append("tls.key"); if (checkFileExists(filepath)) @@ -1162,13 +1386,28 @@ IPropertyTree *queryTlsSecretInfo(const char *name) if (ca) ca->setProp("@path", filepath.str()); } - // TLS TODO: do we want to always require verify, even if no ca ? - verify->setPropBool("@enable", true); + //For now only the "public" issuer implies client certificates are not required + verify->setPropBool("@enable", !strieq(name, "public")); verify->setPropBool("@address_match", false); verify->setPropBool("@accept_selfsigned", false); verify->setProp("trusted_peers", "anyone"); } - return info; + return info.getClear(); +} + +IPropertyTree *getIssuerTlsServerConfigWithTrustedPeers(const char *issuer, const char *trusted_peers) +{ + Owned issuerConfig = getIssuerTlsServerConfig(issuer); + if (!issuerConfig || isEmptyString(trusted_peers)) + return issuerConfig.getClear(); + //TBD: might cache in the future, but needs thought, lookup must include trusted_peers, but will there be cases where trusted_peers can change dynamically? + Owned tlsConfig = createPTreeFromIPT(issuerConfig); + if (!tlsConfig) + return nullptr; + + IPropertyTree *verify = ensurePTree(tlsConfig, "verify"); + verify->setProp("trusted_peers", trusted_peers); + return tlsConfig.getClear(); } enum UseMTLS { UNINIT, DISABLED, ENABLED }; diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index c473cdc0c84..81bada07763 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -22,15 +22,25 @@ #include "jlib.hpp" #include "jstring.hpp" +interface ISecret : extends IInterface +{ + virtual const IPropertyTree * getTree() const = 0; + virtual bool getKeyValue(MemoryBuffer & result, const char * key) const = 0; + virtual bool getKeyValue(StringBuffer & result, const char * key) const = 0; + virtual bool isStale() const = 0; + //Return a sequence number which changes whenever the secret actually changes - so that a caller can determine + //whether it needs to reload the certificates. + virtual unsigned getVersion() const = 0; +}; + extern jlib_decl void setSecretMount(const char * path); extern jlib_decl void setSecretTimeout(unsigned timeoutMs); -extern jlib_decl IPropertyTree *getLocalSecret(const char *category, const char * name); -extern jlib_decl IPropertyTree *getVaultSecret(const char *category, const char *vaultId, const char * name, const char *version=nullptr); -extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name); +extern jlib_decl IPropertyTree *getSecret(const char *category, const char * name, const char * optVaultId = nullptr, const char * optVersion = nullptr); +extern jlib_decl ISecret * resolveSecret(const char *category, const char * name, const char * optRequiredVault); -extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, IPropertyTree *secret, const char * key); -extern jlib_decl bool getSecretKeyValue(StringBuffer & result, IPropertyTree *secret, const char * key); +extern jlib_decl bool getSecretKeyValue(MemoryBuffer & result, const IPropertyTree *secret, const char * key); +extern jlib_decl bool getSecretKeyValue(StringBuffer & result, const IPropertyTree *secret, const char * key); extern jlib_decl bool getSecretValue(StringBuffer & result, const char *category, const char * name, const char * key, bool required); extern jlib_decl void initSecretUdpKey(); @@ -38,12 +48,17 @@ extern jlib_decl const MemoryAttr &getSecretUdpKey(bool required); extern jlib_decl bool containsEmbeddedKey(const char *certificate); -extern jlib_decl IPropertyTree *queryTlsSecretInfo(const char *issuer); -extern jlib_decl IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert=true); +//getIssuerTlsServerConfig must return owned because the internal cache could be updated internally and the return will become invalid, so must be linked +extern jlib_decl IPropertyTree *getIssuerTlsServerConfig(const char *issuer); +extern jlib_decl IPropertyTree *getIssuerTlsServerConfigWithTrustedPeers(const char *issuer, const char *trusted_peers); + +extern jlib_decl IPropertyTree *createIssuerTlsClientConfig(const char *issuer, bool acceptSelfSigned, bool addCACert=true); extern jlib_decl void splitFullUrl(const char *url, bool &https, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &fullpath); extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path); extern jlib_decl void splitUrlIsolateScheme(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &scheme, StringBuffer &hostPort, StringBuffer &path); +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *scheme, const char *userPasswordPair, const char *host, unsigned port, const char *path); +extern jlib_decl StringBuffer &generateDynamicUrlSecretName(StringBuffer &secretName, const char *url, const char *username); extern jlib_decl bool queryMtls(); diff --git a/system/jlib/jsmartsock.cpp b/system/jlib/jsmartsock.cpp index 2c38088d2ea..f7f0b4190dd 100644 --- a/system/jlib/jsmartsock.cpp +++ b/system/jlib/jsmartsock.cpp @@ -216,8 +216,9 @@ CSmartSocketFactory::CSmartSocketFactory(IPropertyTree &service, bool _retry, un throw createSmartSocketException(0, "CSmartSocket factory both name and port required for service configuration"); tlsService = service.getPropBool("@tls"); + issuer.set(service.queryProp("@issuer")); if (tlsService) - tlsConfig.setown(createTlsClientSecretInfo(service.queryProp("@issuer"), !service.getPropBool("@public"), service.getPropBool("@selfSigned"), service.getPropBool("@caCert"))); + tlsConfig.setown(createIssuerTlsClientConfig(issuer, service.getPropBool("@selfSigned"), service.getPropBool("@caCert"))); StringBuffer s; s.append(name).append(':').append(port); @@ -353,7 +354,7 @@ ISocket *CSmartSocketFactory::connect_sock(unsigned timeoutms, SmartSocketEndpoi catch (IException *e) { StringBuffer s("CSmartSocketFactory::connect_sock "); - ep.getUrlStr(s); + ep.getEndpointHostText(s); EXCLOG(e,s.str()); ss->status=false; if (sock) @@ -477,7 +478,7 @@ StringBuffer & CSmartSocketFactory::getUrlStr(StringBuffer &url, bool useHostNam { sep->checkHost(dnsInterval); SocketEndpoint ep = sep->ep; - ep.getUrlStr(url); + ep.getEndpointHostText(url); } } return url; diff --git a/system/jlib/jsmartsock.ipp b/system/jlib/jsmartsock.ipp index 724f639d561..e89c7850d1c 100644 --- a/system/jlib/jsmartsock.ipp +++ b/system/jlib/jsmartsock.ipp @@ -35,7 +35,7 @@ struct SmartSocketEndpoint if (ep.isNull()) throw MakeStringException(-1,"SmartSocketEndpoint resolution failed for '%s' %d",_name,port); StringBuffer ipStr; - ep.getIpText(ipStr); + ep.getHostText(ipStr); if (strcmp(ipStr.str(), _name)!=0) name.append(_name); lastHostUpdate=msTick(); @@ -69,6 +69,7 @@ protected: bool retry; bool tlsService = false; Owned tlsConfig; + StringAttr issuer; unsigned retryInterval; unsigned dnsInterval; @@ -104,6 +105,7 @@ public: virtual StringBuffer & getUrlStr(StringBuffer &str, bool useHostName); virtual bool isTlsService() const override { return tlsService; } virtual const IPropertyTree *queryTlsConfig() const { return tlsConfig; }; + const char *queryTlsIssuer() const { return issuer.str(); } }; diff --git a/system/jlib/jsocket.cpp b/system/jlib/jsocket.cpp index 3b6e4d5b032..f246cc8769f 100644 --- a/system/jlib/jsocket.cpp +++ b/system/jlib/jsocket.cpp @@ -1268,7 +1268,7 @@ int CSocket::name(char *retname,size32_t namemax) if (retname && namemax) { StringBuffer s; - ep.getIpText(s); + ep.getHostText(s); if (namemax-11) { @@ -1545,7 +1545,7 @@ void CSocket::connect_wait(unsigned timems) if (ep.ipequals(targetip)) { unsigned sleeptime = getRandom() % 1000; StringBuffer s; - ep.getIpText(s); + ep.getHostText(s); DBGLOG("Connection to central node %s - sleeping %d milliseconds", s.str(), sleeptime); Sleep(sleeptime); break; @@ -1698,7 +1698,7 @@ void CSocket::setTraceName() { #ifdef _TRACE StringBuffer hostname; - targetip.getIpText(hostname); + targetip.getHostText(hostname); setTraceName("C!", hostname); #endif } @@ -1750,7 +1750,7 @@ int CSocket::logPollError(unsigned revents, const char *rwstr) { StringBuffer errStr; StringBuffer hostname; - targetip.getIpText(hostname); + targetip.getHostText(hostname); errStr.appendf("%s POLLERR %u l:%d r:%s:%d", rwstr, sock, localPort, hostname.str(), hostport); int serror = 0; socklen_t serrlen = sizeof(serror); @@ -2827,7 +2827,7 @@ void CSocket::set_receive_buffer_size(size32_t max) bool CSocket::join_multicast_group(SocketEndpoint &ep) { StringBuffer s; - ep.getIpText(s); // will improve later + ep.getHostText(s); // will improve later MCASTREQ req(s.str()); if (setsockopt(sock, IPPROTO_IP, IP_ADD_MEMBERSHIP,(char*)&req, sizeof(req))!=0) { return false; @@ -2839,7 +2839,7 @@ bool CSocket::join_multicast_group(SocketEndpoint &ep) bool CSocket::leave_multicast_group(SocketEndpoint &ep) { StringBuffer s; - ep.getIpText(s); // will improve later + ep.getHostText(s); // will improve later MCASTREQ req(s.str()); if (setsockopt(sock, IPPROTO_IP, IP_DROP_MEMBERSHIP,(char*)&req, sizeof(req))!=0) { return false; @@ -2867,7 +2867,7 @@ void CSocket::set_ttl(unsigned _ttl) void CSocket::logConnectionInfo(unsigned timeoutms, unsigned conn_mstime) { StringBuffer hostname; - targetip.getIpText(hostname); + targetip.getHostText(hostname); PROGLOG("SOCKTRACE: connect(%u) - time:%u ms fd:%d l:%d r:%s:%d", timeoutms, conn_mstime, sock, localPort, hostname.str(), hostport); // PrintStackReport(); } @@ -2958,7 +2958,7 @@ CSocket::CSocket(const SocketEndpoint &ep,SOCKETMODE smode,const char *name) else { StringBuffer hostname; - targetip.getIpText(hostname); + targetip.getHostText(hostname); setTraceName("S>", hostname.str()); } #endif @@ -3046,7 +3046,7 @@ ISocket* ISocket::multicast_create(unsigned short p, const IpAddress &ip, unsign THROWJSOCKEXCEPTION2(JSOCKERR_bad_address); SocketEndpoint ep(p, ip); StringBuffer tmp; - Owned sock = new CSocket(ep,sm_multicast_server,ip.getIpText(tmp).str()); + Owned sock = new CSocket(ep,sm_multicast_server,ip.getHostText(tmp).str()); sock->open(0,true); if (_ttl) sock->set_ttl(_ttl); @@ -3238,7 +3238,7 @@ const char * GetCachedHostName() if (getInterfaceIp(ip, ifs)) { StringBuffer ips; - ip.getIpText(ips); + ip.getHostText(ips); if (ips.length()) { cachehostname.set(ips.str()); @@ -3634,23 +3634,19 @@ inline char * addbyte(char *s,byte b) *(s++) = b+'0'; return s; } - - StringBuffer & IpAddress::getIpText(StringBuffer & out) const { - if (hostname) - return out.append(hostname); if (::isIp4(netaddr)) { const byte *ip = (const byte *)&netaddr[3]; - char ips[16]; + char ips[16]; char *s = ips; for (unsigned i=0;i<4;i++) { if (i) *(s++) = '.'; s = addbyte(s,ip[i]); } - return out.append(s-ips,ips); + return out.append(s-ips,ips); } char tmp[INET6_ADDRSTRLEN]; const char *res = _inet_ntop(AF_INET6, &netaddr, tmp, sizeof(tmp)); @@ -3659,6 +3655,13 @@ StringBuffer & IpAddress::getIpText(StringBuffer & out) const return out.append(res); } +StringBuffer & IpAddress::getHostText(StringBuffer & out) const +{ + if (hostname) + return out.append(hostname); + return getIpText(out); +} + void IpAddress::ipserialize(MemoryBuffer & out) const { if (((netaddr[2]==0xffff0000)||(netaddr[2]==0))&&(netaddr[1]==0)&&(netaddr[0]==0)) { @@ -3849,12 +3852,28 @@ bool SocketEndpoint::set(const char *name,unsigned short _port) return false; } -void SocketEndpoint::getUrlStr(char * str, size32_t len) const +StringBuffer &SocketEndpoint::getEndpointIpText(StringBuffer &str) const +{ + getIpText(str); + if (port) + str.append(':').append((unsigned)port); // TBD IPv6 put [] on + return str; +} + +StringBuffer &SocketEndpoint::getEndpointHostText(StringBuffer &str) const +{ + getHostText(str); + if (port) + str.append(':').append((unsigned)port); // TBD IPv6 put [] on + return str; +} + +void SocketEndpoint::getEndpointHostText(char * str, size32_t len) const { if (len==0) return; StringBuffer _str; - getUrlStr(_str); + getEndpointHostText(_str); size32_t l = _str.length()+1; if (l>len) { @@ -3864,23 +3883,22 @@ void SocketEndpoint::getUrlStr(char * str, size32_t len) const memcpy(str,_str.str(),l); } -StringBuffer &SocketEndpoint::getUrlStr(StringBuffer &str) const +unsigned SocketEndpoint::hash(unsigned prev) const { - getIpText(str); - if (port) - str.append(':').append((unsigned)port); // TBD IPv6 put [] on - return str; + return hashc((const byte *)&port,sizeof(port),iphash(prev)); } -unsigned SocketEndpoint::hash(unsigned prev) const +StringBuffer &getRemoteAccessibleHostText(StringBuffer &str, const SocketEndpoint &ep) { - return hashc((const byte *)&port,sizeof(port),iphash(prev)); + if (isContainerized()) + return ep.getEndpointIpText(str); + else + return ep.getEndpointHostText(str); } - //--------------------------------------------------------------------------- SocketListCreator::SocketListCreator() @@ -3891,7 +3909,7 @@ SocketListCreator::SocketListCreator() void SocketListCreator::addSocket(const SocketEndpoint &ep) { StringBuffer ipstr; - ep.getIpText(ipstr); + ep.getHostText(ipstr); addSocket(ipstr.str(), ep.port); } @@ -3947,7 +3965,7 @@ void SocketListCreator::addSockets(SocketEndpointArray &array) ForEachItemIn(i,array) { const SocketEndpoint &sockep=array.item(i); StringBuffer ipstr; - sockep.getIpText(ipstr); + sockep.getHostText(ipstr); addSocket(ipstr.str(),sockep.port); } } @@ -6344,7 +6362,7 @@ void multiConnect(const SocketEndpointArray &eps, IPointerArrayOf &rets void failed(unsigned idx,const SocketEndpoint &ep,int err) { StringBuffer s; - PROGLOG("multiConnect failed to %s with %d",ep.getUrlStr(s).str(),err); + PROGLOG("multiConnect failed to %s with %d",ep.getEndpointHostText(s).str(),err); } } notify(retsockets,sect); multiConnect(eps,notify,timeout); @@ -6373,12 +6391,12 @@ StringBuffer &SocketEndpointArray::getText(StringBuffer &text) const if (!count) return text; if (count==1) - return item(0).getUrlStr(text); + return item(0).getEndpointHostText(text); byte lastip[4]; const SocketEndpoint &first = item(0); bool lastis4 = first.getNetAddress(sizeof(lastip),&lastip)==sizeof(lastip); unsigned short lastport = first.port; - first.getIpText(text); + first.getHostText(text); unsigned rep=0; unsigned range=0; for (unsigned i=1;ih_name); else - ip.getIpText(ret); + ip.getHostText(ret); } else - ip.getIpText(ret); + ip.getHostText(ret); return ret; } @@ -7244,7 +7262,7 @@ class CAllowListHandler : public CSimpleInterfaceOf, implemen if (responseText) { responseText->append("Access denied! [server ip="); - queryHostIP().getIpText(*responseText); + queryHostIP().getHostText(*responseText); responseText->append(", client ip="); responseText->append(ip); if (role) diff --git a/system/jlib/jsocket.hpp b/system/jlib/jsocket.hpp index 62a29e666e6..6587e9410ff 100644 --- a/system/jlib/jsocket.hpp +++ b/system/jlib/jsocket.hpp @@ -84,6 +84,9 @@ class jlib_decl IpAddress { unsigned netaddr[4] = { 0, 0, 0, 0 }; StringAttr hostname; // not currently serialized + +protected: + StringBuffer &getHostText(StringBuffer & out, bool ip) const; public: IpAddress() = default; explicit IpAddress(const char *text) { ipset(text); } @@ -99,7 +102,8 @@ class jlib_decl IpAddress bool isLoopBack() const; // is loopback (localhost: 127.0.0.1 or ::1) bool isLocal() const; // matches local interface bool isIp4() const; - StringBuffer &getIpText(StringBuffer & out) const; + StringBuffer &getIpText(StringBuffer &out) const; + StringBuffer &getHostText(StringBuffer & out) const; void ipserialize(MemoryBuffer & out) const; void ipdeserialize(MemoryBuffer & in); unsigned ipdistance(const IpAddress &ip,unsigned offset=0) const; // network order distance (offset: 0-3 word (leat sig.), 0=Ipv4) @@ -159,8 +163,9 @@ class jlib_decl SocketEndpoint : extends IpAddress inline void setLocalHost(unsigned short _port) { port = _port; GetHostIp(*this); } // NB *not* localhost(127.0.0.1) inline void set(unsigned short _port, const IpAddress & _ip) { ipset(_ip); port = _port; }; inline bool equals(const SocketEndpoint &ep) const { return ((port==ep.port)&&ipequals(ep)); } - void getUrlStr(char * str, size32_t len) const; // in form ip4:port or [ip6]:port - StringBuffer &getUrlStr(StringBuffer &str) const; // in form ip4:port or [ip6]:port + StringBuffer &getEndpointIpText(StringBuffer &str) const; + void getEndpointHostText(char * str, size32_t len) const; // in form ip4:port or [ip6]:port + StringBuffer &getEndpointHostText(StringBuffer &str) const; // in form ip4:port or [ip6]:port inline SocketEndpoint & operator = ( const SocketEndpoint &other ) { @@ -178,6 +183,10 @@ class jlib_decl SocketEndpoint : extends IpAddress unsigned short portPadding = 0; }; +// Conditionally return endpoint hostname or resolved IP (may want condition to differ in future, e.g. depending on dns configuration) +// In k8s by default pod hostnames are not resolvable from other pods, use this function when serializing the text of a host to another host +extern jlib_decl StringBuffer &getRemoteAccessibleHostText(StringBuffer &str, const SocketEndpoint &ep); + class jlib_decl SocketEndpointArray : public StructArrayOf { public: diff --git a/system/jlib/jtrace.cpp b/system/jlib/jtrace.cpp index 55fe2056783..278c6a117cd 100644 --- a/system/jlib/jtrace.cpp +++ b/system/jlib/jtrace.cpp @@ -623,12 +623,12 @@ class CTraceManager : implements ITraceManager, public CInterface if (!exportType.isEmpty()) { - if (strcasecmp(exportType.str(), "OS")==0) //To stdout/err + if (stricmp(exportType.str(), "OS")==0) //To stdout/err { exporter = opentelemetry::exporter::trace::OStreamSpanExporterFactory::Create(); DBGLOG("Tracing to stdout/err..."); } - else if (strcasecmp(exportType.str(), "OTLP")==0) + else if (stricmp(exportType.str(), "OTLP")==0) { namespace otlp = opentelemetry::exporter::otlp; @@ -649,9 +649,9 @@ class CTraceManager : implements ITraceManager, public CInterface exporter = otlp::OtlpGrpcExporterFactory::Create(opts); DBGLOG("Tracing to OTLP (%s)", endPoint.str()); } - else if (strcasecmp(exportType.str(), "Prometheus")==0) + else if (stricmp(exportType.str(), "Prometheus")==0) DBGLOG("Tracing to Prometheus currently not supported"); - else if (strcasecmp(exportType.str(), "HPCC")==0) + else if (stricmp(exportType.str(), "HPCC")==0) DBGLOG("Tracing to HPCC JLog currently not supported"); } else @@ -841,4 +841,4 @@ void initTraceManager(const char * componentName, IPropertyTree * config) ITraceManager & queryTraceManager() { return *theTraceManager.query([] () { return new CTraceManager; }); //throws if not initialized -} \ No newline at end of file +} diff --git a/system/mp/mpbase.cpp b/system/mp/mpbase.cpp index 086995fc54a..c5d34f76867 100644 --- a/system/mp/mpbase.cpp +++ b/system/mp/mpbase.cpp @@ -542,7 +542,7 @@ protected: friend class CNodeIterator; if (!count) return text; if (count==1) - return nodes[0]->endpoint().getUrlStr(text); + return nodes[0]->endpoint().getEndpointHostText(text); // following is rather slow maybe could do with more direct method with pointers TBD SocketEndpointArray epa; for(unsigned i=0;iget(s); - return s.getUrlStr(val); + return s.getEndpointHostText(val); } }; @@ -231,12 +231,12 @@ class DECL_EXCEPTION CMPException: public IMP_Exception, public CInterface StringBuffer tmp; switch (error) { case MPERR_ok: str.append("OK"); break; - case MPERR_connection_failed: str.appendf("MP connect failed (%s)",endpoint.getUrlStr(tmp).str()); break; + case MPERR_connection_failed: str.appendf("MP connect failed (%s)",endpoint.getEndpointHostText(tmp).str()); break; case MPERR_process_not_in_group: str.appendf("Current process not in Communicator group"); break; - case MPERR_protocol_version_mismatch: str.appendf("Protocol version mismatch (%s)",endpoint.getUrlStr(tmp).str()); break; + case MPERR_protocol_version_mismatch: str.appendf("Protocol version mismatch (%s)",endpoint.getEndpointHostText(tmp).str()); break; // process crashes (segv, etc.) often cause this exception which is logged and can be misleading // change it from "MP link closed" to something more helpful - case MPERR_link_closed: str.appendf("Unexpected process termination (ep:%s)",endpoint.getUrlStr(tmp).str()); break; + case MPERR_link_closed: str.appendf("Unexpected process termination (ep:%s)",endpoint.getEndpointHostText(tmp).str()); break; } return str; } @@ -766,7 +766,7 @@ void traceSlowReadTms(const char *msg, ISocket *sock, void *dst, size32_t minSiz { SocketEndpoint ep; sock->getPeerEndpoint(ep); - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); } WARNLOG("%s %s, stalled for %d ms so far", msg, epStr.str(), elapsedMs); } @@ -779,7 +779,7 @@ void traceSlowReadTms(const char *msg, ISocket *sock, void *dst, size32_t minSiz { SocketEndpoint ep; sock->getPeerEndpoint(ep); - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); } WARNLOG("%s %s, took: %d ms", msg, epStr.str(), readTmsTimer.elapsedMs()); } @@ -879,7 +879,7 @@ protected: friend class CMPPacketReader; { StringBuffer str; #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: connecting to %s role: %" I64F "u", remoteep.getUrlStr(str).str(), parent->getRole()); + LOG(MCdebugInfo, unknownJob, "MP: connecting to %s role: %" I64F "u", remoteep.getEndpointHostText(str).str(), parent->getRole()); #endif if (((int)tm.timeout)<0) remaining = CONNECT_TIMEOUT; @@ -931,9 +931,9 @@ protected: friend class CMPPacketReader; #ifdef _FULLTRACE StringBuffer tmp1; - connectHdr.id[0].getUrlStr(tmp1); + connectHdr.id[0].getEndpointHostText(tmp1); tmp1.append(' '); - connectHdr.id[1].getUrlStr(tmp1); + connectHdr.id[1].getEndpointHostText(tmp1); LOG(MCdebugInfo, unknownJob, "MP: connect after socket write %s",tmp1.str()); #endif @@ -1028,7 +1028,7 @@ protected: friend class CMPPacketReader; e->Release(); #ifdef _TRACE - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).str(),retrycount+1); + LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } else @@ -1038,7 +1038,7 @@ protected: friend class CMPPacketReader; { SocketEndpoint ep; newsock->getPeerEndpoint(ep); - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); } WARNLOG("MP: connect to: %s, stalled for %d ms so far", epStr.str(), msTick()-startMs); e->Release(); @@ -1074,7 +1074,7 @@ protected: friend class CMPPacketReader; { SocketEndpoint ep; newsock->getPeerEndpoint(ep); - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); } StringBuffer allowExcStr; exitException.setown(makeStringExceptionV(-99, "Error '%s' reading Allowlist exception from: %s", e->errorMessage(allowExcStr).str(), epStr.str())); @@ -1093,7 +1093,7 @@ protected: friend class CMPPacketReader; { SocketEndpoint ep; newsock->getPeerEndpoint(ep); - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); } WARNLOG("MP: connect to: %s, took: %d ms", epStr.str(), elapsedMs); } @@ -1132,7 +1132,7 @@ protected: friend class CMPPacketReader; #ifdef _TRACE StringBuffer str; str.clear(); - LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getUrlStr(str).str(),retrycount+1); + LOG(MCdebugInfo, unknownJob, "MP: Retrying connection to %s, %d attempts left",remoteep.getEndpointHostText(str).str(),retrycount+1); #endif } @@ -1221,7 +1221,7 @@ protected: friend class CMPPacketReader; #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getUrlStr(ep1).str(),hdrsize,hdr2size,bodysize); + LOG(MCdebugInfo, unknownJob, "WritePacket(target=%s,(%d,%d,%d))",remoteep.getEndpointHostText(ep1).str(),hdrsize,hdr2size,bodysize); unsigned t2 = msTick(); #endif unsigned n = 0; @@ -1282,7 +1282,7 @@ protected: friend class CMPPacketReader; return true; } StringBuffer ep; - remoteep.getUrlStr(ep); + remoteep.getEndpointHostText(ep); for (;;) { CTimeMon pingtm(1000*60); if (sendPing(pingtm)) @@ -1375,7 +1375,7 @@ protected: friend class CMPPacketReader; StringBuffer & queryEpStr(StringBuffer &s) { - return remoteep.getUrlStr(s); + return remoteep.getEndpointHostText(s); } bool isClosed() @@ -1411,7 +1411,7 @@ class UserPacketHandler // default #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getUrlStr(ep1).str(),hdr.sender.getUrlStr(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, unknownJob, "MP: send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif return channel->writepacket(&hdr,sizeof(hdr),mb.toByteArray(),mb.length(),tm); } @@ -1466,7 +1466,7 @@ class MultiPacketHandler // TAG_SYS_MULTI if ((ms-lastErrMs) > 1000) // avoid logging too much { StringBuffer errorMsg("sender="); - msg.getSender().getUrlStr(errorMsg).newline(); + msg.getSender().getEndpointHostText(errorMsg).newline(); errorMsg.append("This header: "); mhdr.getDetails(errorMsg).newline(); if (otherMhdr) @@ -1543,7 +1543,7 @@ class MultiPacketHandler // TAG_SYS_MULTI #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getUrlStr(ep1).str(),hdr.sender.getUrlStr(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, unknownJob, "MP: multi-send(target=%s,sender=%s,tag=%d,replytag=%d,size=%d)",hdr.target.getEndpointHostText(ep1).str(),hdr.sender.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif PacketHeader outhdr; outhdr = hdr; @@ -1697,7 +1697,7 @@ class CMPPacketReader: public ISocketSelectNotify, public CInterface #ifdef _FULLTRACE StringBuffer ep1; StringBuffer ep2; - LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getUrlStr(ep1).str(),hdr.target.getUrlStr(ep2).str(),hdr.tag,hdr.replytag,hdr.size); + LOG(MCdebugInfo, unknownJob, "MP: ReadPacket(sender=%s,target=%s,tag=%d,replytag=%d,size=%d)",hdr.sender.getEndpointHostText(ep1).str(),hdr.target.getEndpointHostText(ep2).str(),hdr.tag,hdr.replytag,hdr.size); #endif remaining = hdr.size-sizeof(hdr); activemsg = new CMessageBuffer(remaining); // will get from low level IO at some stage @@ -1847,8 +1847,8 @@ bool CMPChannel::attachSocket(ISocket *newsock,const SocketEndpoint &_remoteep,c StringBuffer ep1; StringBuffer ep2; - _localep.getUrlStr(ep1); - _remoteep.getUrlStr(ep2); + _localep.getEndpointHostText(ep1); + _remoteep.getEndpointHostText(ep2); LOG(MCdebugInfo, unknownJob, "MP: Possible clash between %s->%s %d(%d)",ep1.str(),ep2.str(),(int)ismaster,(int)master); try { @@ -2132,7 +2132,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list #if defined(_USE_OPENSSL) if (parent->useTLS) - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); #endif } @@ -2225,7 +2225,7 @@ int CMPConnectThread::run() StringBuffer s; SocketEndpoint ep1; sock->getPeerEndpoint(ep1); - PROGLOG("MP: Connect Thread: socket accepted from %s",ep1.getUrlStr(s).str()); + PROGLOG("MP: Connect Thread: socket accepted from %s",ep1.getEndpointHostText(s).str()); #endif sock->set_keep_alive(true); @@ -2263,7 +2263,7 @@ int CMPConnectThread::run() { // not sure how to get here as this is not one of the possible outcomes of above: rd == 0 or rd == sizeof(id) or an exception StringBuffer errMsg("MP Connect Thread: invalid number of connection bytes serialized from "); - peerEp.getUrlStr(errMsg); + peerEp.getEndpointHostText(errMsg); FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); sock->close(); continue; @@ -2273,7 +2273,7 @@ int CMPConnectThread::run() if (allowListCallback) { StringBuffer ipStr; - peerEp.getIpText(ipStr); + peerEp.getHostText(ipStr); StringBuffer responseText; // filled if denied, NB: if amount sent is > sizeof(ConnectHdr) we can differentiate exception from success if (!allowListCallback->isAllowListed(ipStr, connectHdr.getRole(), &responseText)) { @@ -2320,14 +2320,14 @@ int CMPConnectThread::run() { // JCSMORE, I think _remoteep really must/should match a IP of this local host errMsg.append("MP Connect Thread: invalid remote and/or host ep serialized from "); - peerEp.getUrlStr(errMsg); + peerEp.getEndpointHostText(errMsg); FLLOG(MCoperatorWarning, unknownJob, "%s", errMsg.str()); } else if (parent->mpTraceLevel >= MPVerboseMsgThreshold) { // all zeros msg received errMsg.append("MP Connect Thread: connect with empty msg received, assumed port monitor check from "); - peerEp.getUrlStr(errMsg); + peerEp.getEndpointHostText(errMsg); PROGLOG("%s", errMsg.str()); } sock->close(); @@ -2335,9 +2335,9 @@ int CMPConnectThread::run() } #ifdef _FULLTRACE StringBuffer tmp1; - _remoteep.getUrlStr(tmp1); + _remoteep.getEndpointHostText(tmp1); tmp1.append(' '); - hostep.getUrlStr(tmp1); + hostep.getEndpointHostText(tmp1); PROGLOG("MP: Connect Thread: after read %s",tmp1.str()); #endif checkSelfDestruct(&connectHdr.id[0],sizeof(connectHdr.id)); @@ -2353,7 +2353,7 @@ int CMPConnectThread::run() #ifdef _TRACE StringBuffer str1; StringBuffer str2; - LOG(MCdebugInfo, unknownJob, "MP Connect Thread: connected to %s",_remoteep.getUrlStr(str1).str()); + LOG(MCdebugInfo, unknownJob, "MP Connect Thread: connected to %s",_remoteep.getEndpointHostText(str1).str()); #endif } #ifdef _FULLTRACE @@ -2738,7 +2738,7 @@ void CMPServer::notifyClosed(SocketEndpoint &ep, bool trace) if (trace) { StringBuffer url; - LOG(MCdebugInfo, unknownJob, "MP: CMPServer::notifyClosed %s",ep.getUrlStr(url).str()); + LOG(MCdebugInfo, unknownJob, "MP: CMPServer::notifyClosed %s",ep.getEndpointHostText(url).str()); PrintStackReport(); } #endif diff --git a/system/mp/mplog.cpp b/system/mp/mplog.cpp index c9d90ccc59b..efb0e3765e5 100644 --- a/system/mp/mplog.cpp +++ b/system/mp/mplog.cpp @@ -368,7 +368,7 @@ void LinkToParentLogMsgHandler::addToPTree(IPropertyTree * tree) const IPropertyTree * handlerTree = createPTree(ipt_caseInsensitive); handlerTree->setProp("@type", "linktoparent"); StringBuffer buff; - parentNode->endpoint().getUrlStr(buff); + parentNode->endpoint().getEndpointHostText(buff); handlerTree->setProp("@url", buff.str()); tree->addPropTree("handler", handlerTree); } diff --git a/system/mp/test/mptest.cpp b/system/mp/test/mptest.cpp index c3ed6b45ad7..23ffd37f714 100644 --- a/system/mp/test/mptest.cpp +++ b/system/mp/test/mptest.cpp @@ -309,7 +309,7 @@ void Test6(IGroup *group,ICommunicator *comm) StringAttr str; mb.read(str); StringBuffer url; - PROGLOG("(6) Received '%s' from %s",str.get(),mb.getSender().getUrlStr(url).str()); + PROGLOG("(6) Received '%s' from %s",str.get(),mb.getSender().getEndpointHostText(url).str()); } else if (group->rank()==0) { @@ -452,7 +452,7 @@ void MultiTest(ICommunicator *_comm) break; PROGLOG("MPTEST: MultiTest server Received from %u, len = %u",rr, mb.length()); StringBuffer str; - comm->queryGroup().queryNode(rr).endpoint().getUrlStr(str); + comm->queryGroup().queryNode(rr).endpoint().getEndpointHostText(str); // PROGLOG("MPTEST: MultiTest server Received from %s",str.str()); buff->deserialize(mb); @@ -524,7 +524,7 @@ void MultiTest(ICommunicator *_comm) #if 0 StringBuffer str; - comm->queryGroup().queryNode(targets[n]).endpoint().getUrlStr(str); + comm->queryGroup().queryNode(targets[n]).endpoint().getEndpointHostText(str); PROGLOG("MPTEST: Multitest client Sending to %s, length=%u",str.str(), mb.length()); #endif @@ -1721,7 +1721,7 @@ int main(int argc, char* argv[]) for (rank_t k=0;kgetProp(".//@ldapAdminVaultId", vaultId);//optional HashiCorp vault ID DBGLOG("Retrieving LDAP Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree; - if (!vaultId.isEmpty()) - secretTree.setown(getVaultSecret("authn", vaultId, adminUserSecretKey.str(), nullptr)); - else - secretTree.setown(getSecret("authn", adminUserSecretKey.str())); + + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (!secretTree) throw MakeStringException(-1, "Error retrieving LDAP Admin username/password"); @@ -497,11 +494,7 @@ class CLdapConfig : implements ILdapConfig, public CInterface cfg->getProp(".//@hpccAdminVaultId", vaultId);//optional HashiCorp vault ID DBGLOG("Retrieving optional HPCC Admin username/password from secrets repo: %s %s", !vaultId.isEmpty() ? vaultId.str() : "", adminUserSecretKey.str()); - Owned secretTree; - if (!vaultId.isEmpty()) - secretTree.setown(getVaultSecret("authn", vaultId, adminUserSecretKey.str(), nullptr)); - else - secretTree.setown(getSecret("authn", adminUserSecretKey.str())); + Owned secretTree(getSecret("authn", adminUserSecretKey.str(), vaultId, nullptr)); if (secretTree) { getSecretKeyValue(m_HPCCAdminUser_username, secretTree, "username"); diff --git a/system/security/securesocket/securesocket.cpp b/system/security/securesocket/securesocket.cpp index 37285487129..2b416916d7d 100644 --- a/system/security/securesocket/securesocket.cpp +++ b/system/security/securesocket/securesocket.cpp @@ -629,10 +629,10 @@ bool CSecureSocket::verify_cert(X509* cert) SocketEndpoint ep; m_socket->getPeerEndpoint(ep); StringBuffer iptxt; - ep.getIpText(iptxt); + ep.getHostText(iptxt); SocketEndpoint cnep(cn.str()); StringBuffer cniptxt; - cnep.getIpText(cniptxt); + cnep.getHostText(cniptxt); DBGLOG("peer ip=%s, certificate ip=%s", iptxt.str(), cniptxt.str()); if(!(cniptxt.length() > 0 && stricmp(iptxt.str(), cniptxt.str()) == 0)) { @@ -1985,9 +1985,9 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocket return new securesocket::CSecureSocketContext(ssf->queryTlsConfig(), ClientSocket); } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType sockettype) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *issuer, SecureSocketType sockettype) { - IPropertyTree *info = queryTlsSecretInfo(mtlsSecretName); + Owned info = getIssuerTlsServerConfig(issuer); //if the secret doesn't exist doesn't exist just go on without it. IF it is required the tls connection will fail. //This is primarily for client side... server side would probably use the explict ptree config or explict cert param at least for now. if (info) @@ -1996,16 +1996,16 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const cha return createSecureSocketContext(sockettype); } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, bool requireMtlsFlag) { - if (!queryMtls()) + if (requireMtlsFlag && !queryMtls()) throw makeStringException(-100, "TLS secure communication requested but not configured"); - IPropertyTree *info = queryTlsSecretInfo(mtlsSecretName); - if (info) - return createSecureSocketContextEx2(info, ServerSocket); - else + Owned info = getIssuerTlsServerConfig(issuer); + if (!info) throw makeStringException(-101, "TLS secure communication requested but not configured (2)"); + + return createSecureSocketContextEx2(info, ServerSocket); } SECURESOCKET_API ICertificate *createCertificate() @@ -2172,7 +2172,7 @@ class CSingletonSecureSocketConnection: public CSingletonSocketConnection state = Snone; cancelling = false; secureContextClient.setown(createSecureSocketContextSecret("local", ClientSocket)); - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); #ifdef _CONTAINERIZED tlsLogLevel = getComponentConfigSP()->getPropInt("logging/@detail", SSLogMin); if (tlsLogLevel >= ExtraneousMsgThreshold) // or InfoMsgThreshold ? diff --git a/system/security/securesocket/securesocket.hpp b/system/security/securesocket/securesocket.hpp index cd3001b8460..2bf1758d92e 100644 --- a/system/security/securesocket/securesocket.hpp +++ b/system/security/securesocket/securesocket.hpp @@ -92,7 +92,7 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextEx(const char* c SECURESOCKET_API ISecureSocketContext* createSecureSocketContextEx2(const IPropertyTree* config, SecureSocketType); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocketFactory* ssf); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType); -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName); +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, bool requireMtlsConfig); SECURESOCKET_API ICertificate *createCertificate(); SECURESOCKET_API int signCertificate(const char* csr, const char* ca_certificate, const char* ca_privkey, const char* ca_passphrase, int days, StringBuffer& certificate); }; diff --git a/testing/regress/ecl-test-azure.json b/testing/regress/ecl-test-azure.json index 5877e8a355c..487d500e492 100644 --- a/testing/regress/ecl-test-azure.json +++ b/testing/regress/ecl-test-azure.json @@ -50,10 +50,14 @@ ], "Params":[ "PassTest.ecl:bla='A value'", - "httpcall_multiheader.ecl:TargetIP=40.88.243.151", - "httpcall_jsonpost.ecl:targetIP=52.226.177.210", - "httpcall_xmlpost.ecl:targetIP=52.226.177.210", - "soapcall_multihttpheader.ecl:TargetIP=40.88.243.151" + "httpcall_multiheader.ecl:TargetIP=eclwatch", + "httpcall_jsonpost.ecl:targetIP=roxie", + "httpcall_xmlpost.ecl:targetIP=roxie", + "soapcall_multihttpheader.ecl:TargetIP=eclwatch", + "soapcall.ecl:targetIP=roxie", + "roxiegzip.ecl:TargetIP=roxie", + "roxiewhitespace.ecl:TargetIP=roxie", + "roxiepipe.ecl:TargetIP=roxie" ], "engineParams":[ "failOnLeaks", diff --git a/testing/regress/ecl/httpcall_jsonpost.ecl b/testing/regress/ecl/httpcall_jsonpost.ecl index 7a3c08e4399..5af5591ef07 100644 --- a/testing/regress/ecl/httpcall_jsonpost.ecl +++ b/testing/regress/ecl/httpcall_jsonpost.ecl @@ -15,12 +15,10 @@ limitations under the License. ############################################################################## */ -//class=spray //nothor //nohthor //class=roxieserviceaccess -//version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875' #option('generateGlobalId', true); diff --git a/testing/regress/ecl/httpcall_xmlpost.ecl b/testing/regress/ecl/httpcall_xmlpost.ecl index 73ae15866d9..74b08ed969f 100644 --- a/testing/regress/ecl/httpcall_xmlpost.ecl +++ b/testing/regress/ecl/httpcall_xmlpost.ecl @@ -15,12 +15,10 @@ limitations under the License. ############################################################################## */ -//class=spray //nothor //nohthor //class=roxieserviceaccess -//version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875' #option('generateGlobalId', true); diff --git a/testing/regress/ecl/roxiegzip.ecl b/testing/regress/ecl/roxiegzip.ecl index 0c7f3406da0..2ae23f5e543 100644 --- a/testing/regress/ecl/roxiegzip.ecl +++ b/testing/regress/ecl/roxiegzip.ecl @@ -1,6 +1,26 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + //nothor //nohthor +string TargetIP := '.' : stored('TargetIP'); +string TargetURL := 'http://' + TargetIP + ':9876'; + NameRec := RECORD string First; string Last; @@ -72,9 +92,6 @@ roxieEchoTestResponseRecord doFail() := TRANSFORM self.Exception.Source := 'Test'; END; -string TargetIP := '.' : stored('TargetIP'); -string TargetURL := 'http://' + TargetIP + ':9876'; - gzipResult := SOAPCALL(TargetURL, 'roxie_echo', roxieEchoTestRequestRecord, DATASET(roxieEchoTestResponseRecord), LITERAL, diff --git a/testing/regress/ecl/roxiepipe.ecl b/testing/regress/ecl/roxiepipe.ecl index 73762b90985..e9f39b9c49a 100644 --- a/testing/regress/ecl/roxiepipe.ecl +++ b/testing/regress/ecl/roxiepipe.ecl @@ -1,6 +1,25 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + //nothor //nohthor +string TargetIP := '.' : stored('TargetIP'); + NameRec := RECORD string10 First; string15 Last; @@ -49,7 +68,7 @@ pipe_recv := PIPE(pipe_send, ' -ow ' + SIZEOF(PersonRec) + ' -b 3' + ' -mr 2' + - ' -h .:9876 ' + + ' -h ' + TargetIP + ':9876 ' + ' -r Peeps' + ' -q ""' , PersonRec); diff --git a/testing/regress/ecl/roxiewhitespace.ecl b/testing/regress/ecl/roxiewhitespace.ecl index 64edfe8a51e..df007a4ca67 100644 --- a/testing/regress/ecl/roxiewhitespace.ecl +++ b/testing/regress/ecl/roxiewhitespace.ecl @@ -1,6 +1,26 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + //nothor //nohthor +string TargetIP := '.' : stored('TargetIP'); +string TargetURL := 'http://' + TargetIP + ':9876'; + NameRec := RECORD string First; string Last; @@ -46,9 +66,6 @@ roxieEchoTestResponseRecord doFail() := TRANSFORM self.Exception.Source := 'Test'; END; -string TargetIP := '.' : stored('TargetIP'); -string TargetURL := 'http://' + TargetIP + ':9876'; - soapcallResult := SOAPCALL(TargetURL, 'roxie_keepwhitespace', roxieEchoTestRequestRecord, DATASET(roxieEchoTestResponseRecord), LITERAL, diff --git a/testing/regress/ecl/soapcall.ecl b/testing/regress/ecl/soapcall.ecl index b42187fe8d8..9b1e9c195cb 100644 --- a/testing/regress/ecl/soapcall.ecl +++ b/testing/regress/ecl/soapcall.ecl @@ -16,7 +16,6 @@ ############################################################################## */ //class=roxieserviceaccess -//version targetIP='127.0.0.1',goodPort='9876',blacListedPort='9875' #option('generateGlobalId', true); diff --git a/testing/regress/ecl/soapcall_multihttpheader.ecl b/testing/regress/ecl/soapcall_multihttpheader.ecl index dff26e1359b..db81cab2d34 100644 --- a/testing/regress/ecl/soapcall_multihttpheader.ecl +++ b/testing/regress/ecl/soapcall_multihttpheader.ecl @@ -1,3 +1,20 @@ +/*############################################################################## + + HPCC SYSTEMS software Copyright (C) 2023 HPCC Systems®. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +############################################################################## */ + string TargetIP := '.' : stored('TargetIP'); string storedHeader := 'StoredHeaderDefault' : stored('storedHeader'); diff --git a/testing/unittests/dalitests.cpp b/testing/unittests/dalitests.cpp index 4d0e7fac020..b74c13dbf9e 100644 --- a/testing/unittests/dalitests.cpp +++ b/testing/unittests/dalitests.cpp @@ -107,7 +107,7 @@ void checkFilePart(IChecker *checker,IDistributedFilePart *part,bool blocked) for (unsigned copy=0;copyqueryNode(copy); if (node) - checker->add(copy,"queryNode",node->endpoint().getUrlStr(tmp.clear()).str()); + checker->add(copy,"queryNode",node->endpoint().getEndpointHostText(tmp.clear()).str()); else checker->error("missing node"); checker->add(copy,"getFilename",part->getFilename(rfn,copy).getRemotePath(tmp.clear()).str()); @@ -1773,7 +1773,7 @@ static IFileDescriptor *createDescriptor(const char* dir, const char* name, unsi SocketEndpoint ep; ep.setLocalHost(0); StringBuffer ip; - ep.getIpText(ip); + ep.getHostText(ip); for (unsigned k=0;k node = createINode(s.str()); diff --git a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp index 567a270b6db..5abef028f6f 100644 --- a/thorlcr/activities/hashdistrib/thhashdistribslave.cpp +++ b/thorlcr/activities/hashdistrib/thhashdistribslave.cpp @@ -1247,7 +1247,7 @@ class CDistributorBase : implements IHashDistributor, implements IExceptionHandl catch (IException *e) { StringBuffer senderStr; - activity->queryContainer().queryJob().queryJobGroup().queryNode(n+1).endpoint().getUrlStr(senderStr); + activity->queryContainer().queryJob().queryJobGroup().queryNode(n+1).endpoint().getEndpointHostText(senderStr); IException *e2 = MakeActivityException(activity, e, "Received from node: %s", senderStr.str()); e->Release(); throw e2; diff --git a/thorlcr/activities/result/thresult.cpp b/thorlcr/activities/result/thresult.cpp index 83d1c5fd60f..6322880343f 100644 --- a/thorlcr/activities/result/thresult.cpp +++ b/thorlcr/activities/result/thresult.cpp @@ -57,7 +57,7 @@ class CResultActivityMaster : public CMasterActivity CMessageBuffer mb; if (abortSoon || !receiveMsg(mb, RANK_ALL, replyTag, NULL)) break; StringBuffer str; - mb.getSender().getUrlStr(str); + mb.getSender().getEndpointHostText(str); size32_t sz; mb.read(sz); if (sz) diff --git a/thorlcr/graph/thgraphmaster.cpp b/thorlcr/graph/thgraphmaster.cpp index fe7ed508848..696472f604f 100644 --- a/thorlcr/graph/thgraphmaster.cpp +++ b/thorlcr/graph/thgraphmaster.cpp @@ -290,7 +290,7 @@ void CSlaveMessageHandler::threadmain() Owned e = deserializeThorException(msg); e->setSlave(slave); StringBuffer tmpStr("Slave "); - job.queryJobGroup().queryNode(slave).endpoint().getUrlStr(tmpStr); + job.queryJobGroup().queryNode(slave).endpoint().getEndpointHostText(tmpStr); GraphPrintLog(graph, e, "%s", tmpStr.append(": slave initialization error").str()); throw e.getClear(); } @@ -2530,7 +2530,7 @@ void CMasterGraph::sendActivityInitData() if (!queryJobChannel().queryJobComm().send(msg, w+1, mpTag, LONGTIMEOUT)) { StringBuffer epStr; - throw MakeStringException(0, "Timeout sending to slave %s", job.querySlaveGroup().queryNode(w).endpoint().getUrlStr(epStr).str()); + throw MakeStringException(0, "Timeout sending to slave %s", job.querySlaveGroup().queryNode(w).endpoint().getEndpointHostText(epStr).str()); } ++sentTo; } @@ -2557,7 +2557,7 @@ void CMasterGraph::sendActivityInitData() if (!e.get()) { StringBuffer tmpStr("Slave "); - queryJob().queryJobGroup().queryNode(sender).endpoint().getUrlStr(tmpStr); + queryJob().queryJobGroup().queryNode(sender).endpoint().getEndpointHostText(tmpStr); GraphPrintLog(se, "%s", tmpStr.append(": slave initialization error").str()); e.setown(se.getClear()); } diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index cf6aa750fa5..afcc66595aa 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1795,7 +1795,7 @@ void CJobSlave::startJob() SocketEndpoint ep; ep.setLocalHost(0); StringBuffer s; - throw MakeThorException(TE_NotEnoughFreeSpace, "Node %s has %u MB(s) of available disk space, specified minimum for this job: %u MB(s)", ep.getUrlStr(s).str(), (unsigned) freeSpace / 0x100000, minFreeSpace); + throw MakeThorException(TE_NotEnoughFreeSpace, "Node %s has %u MB(s) of available disk space, specified minimum for this job: %u MB(s)", ep.getEndpointHostText(s).str(), (unsigned) freeSpace / 0x100000, minFreeSpace); } } queryThor().queryKeyedJoinService().setCurrentJob(*this); diff --git a/thorlcr/master/mawatchdog.cpp b/thorlcr/master/mawatchdog.cpp index 153fe2e34b5..5b756d168f2 100644 --- a/thorlcr/master/mawatchdog.cpp +++ b/thorlcr/master/mawatchdog.cpp @@ -54,7 +54,7 @@ class CMachineStatus { markdead = false; StringBuffer epstr; - ep.getUrlStr(epstr); + ep.getEndpointHostText(epstr); LOG(MCdebugProgress, thorJob, "Watchdog : Marking Machine as Up! [%s]", epstr.str()); } } @@ -149,7 +149,7 @@ void CMasterWatchdogBase::checkMachineStatus() if (!mstate->alive) { StringBuffer epstr; - mstate->ep.getUrlStr(epstr); + mstate->ep.getEndpointHostText(epstr); if (mstate->markdead) abortThor(MakeThorOperatorException(TE_AbortException, "Watchdog has lost contact with Thor slave: %s (Process terminated or node down?)", epstr.str()), TEC_Watchdog); else @@ -224,7 +224,7 @@ void CMasterWatchdogBase::threadmain() else { StringBuffer epstr; - hb.sender.getUrlStr(epstr); + hb.sender.getEndpointHostText(epstr); LOG(MCdebugProgress, thorJob, "Watchdog : Unknown Machine! [%s]", epstr.str()); //TBD } } @@ -251,7 +251,7 @@ void CMasterWatchdogBase::threadmain() { const SocketEndpoint &ep = e->queryEndpoint(); StringBuffer epStr; - ep.getUrlStr(epStr); + ep.getEndpointHostText(epStr); abortThor(MakeThorOperatorException(TE_AbortException, "Watchdog has lost connectivity with Thor slave: %s (Process terminated or node down?)", epStr.str()), TEC_Watchdog); } } @@ -301,7 +301,7 @@ class CMasterWatchdogUDP : public CMasterWatchdogBase { SocketEndpoint masterEp(getMasterPortBase()); StringBuffer ipStr; - masterEp.getIpText(ipStr); + masterEp.getHostText(ipStr); Owned sock = ISocket::udp_connect(getFixedPort(masterEp.port, TPORT_watchdog), ipStr.str()); // send empty packet, stopped set, will cease reading HeartBeatPacketHeader hb; diff --git a/thorlcr/master/thgraphmanager.cpp b/thorlcr/master/thgraphmanager.cpp index 771ad01c38a..29d31081e68 100644 --- a/thorlcr/master/thgraphmanager.cpp +++ b/thorlcr/master/thgraphmanager.cpp @@ -456,7 +456,7 @@ bool CJobManager::execute(IConstWorkUnit *workunit, const char *wuid, const char { WorkunitUpdate wu(&workunit->lock()); StringBuffer sb; - queryHostIP().getIpText(sb); + queryHostIP().getHostText(sb); wu->setDebugAgentListenerIP(sb); //tells debugger what IP to write commands to wu->setDebugAgentListenerPort(debugListener->getPort()); } @@ -587,7 +587,7 @@ void CJobManager::run() conversation.clear(); SocketEndpoint masterEp(getMasterPortBase()); StringBuffer url; - PROGLOG("ThorLCR(%s) available, waiting on queue %s",masterEp.getUrlStr(url).str(),queueName.get()); + PROGLOG("ThorLCR(%s) available, waiting on queue %s",masterEp.getEndpointHostText(url).str(),queueName.get()); struct CLock { @@ -787,7 +787,7 @@ bool CJobManager::doit(IConstWorkUnit *workunit, const char *graphName, const So StringAttr wuid(workunit->queryWuid()); StringAttr user(workunit->queryUser()); - LOG(MCdebugInfo, thorJob, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getUrlStr(s).str()); + LOG(MCdebugInfo, thorJob, "Processing wuid=%s, graph=%s from agent: %s", wuid.str(), graphName, agentep.getEndpointHostText(s).str()); LOG(MCauditInfo,",Progress,Thor,Start,%s,%s,%s,%s,%s,%s", queryServerStatus().queryProperties()->queryProp("@thorname"), wuid.str(), @@ -882,7 +882,7 @@ void CJobManager::reply(IConstWorkUnit *workunit, const char *wuid, IException * else s.append("Posting OK"); s.append(" to agent "); - agentep.getUrlStr(s); + agentep.getEndpointHostText(s); s.append(" for workunit(").append(wuid).append(")"); PROGLOG("%s", s.str()); MemoryBuffer replyMb; @@ -924,7 +924,7 @@ void CJobManager::reply(IConstWorkUnit *workunit, const char *wuid, IException * replyMb.append((unsigned)DAMP_THOR_REPLY_GOOD); if (!conversation->send(replyMb)) { s.clear(); - IERRLOG("Failed to reply to agent %s",agentep.getUrlStr(s).str()); + IERRLOG("Failed to reply to agent %s",agentep.getEndpointHostText(s).str()); } conversation.clear(); handlingConversation = false; @@ -1359,7 +1359,7 @@ void thorMain(ILogMsgHandler *logHandler, const char *wuid, const char *graphNam thorQueue->connect(false); } - queryMyNode()->endpoint().getUrlStr(instance); + queryMyNode()->endpoint().getEndpointHostText(instance); StringBuffer currentGraphName(graphName); StringBuffer currentWuid(wuid); diff --git a/thorlcr/master/thmastermain.cpp b/thorlcr/master/thmastermain.cpp index 99f38596b4f..f01925ab183 100644 --- a/thorlcr/master/thmastermain.cpp +++ b/thorlcr/master/thmastermain.cpp @@ -181,7 +181,7 @@ class CRegistryServer : public CSimpleInterface rank_t sender = queryNodeGroup().rank(senderNode); SocketEndpoint ep = senderNode->endpoint(); StringBuffer url; - ep.getUrlStr(url); + ep.getEndpointHostText(url); if (RANK_NULL == sender) { PROGLOG("Node %s trying to deregister is not part of this cluster", url.str()); @@ -235,7 +235,7 @@ class CRegistryServer : public CSimpleInterface { const SocketEndpoint &ep = queryNodeGroup().queryNode(slave+1).endpoint(); StringBuffer url; - ep.getUrlStr(url); + ep.getEndpointHostText(url); if (!status->test(slave)) { PROGLOG("Slave %d (%s) trying to unregister, but not currently registered", slave+1, url.str()); @@ -252,7 +252,7 @@ class CRegistryServer : public CSimpleInterface { SocketEndpoint ep = queryNodeGroup().queryNode(slave+1).endpoint(); StringBuffer url; - ep.getUrlStr(url); + ep.getEndpointHostText(url); if (status->test(slave)) { PROGLOG("Slave %d (%s) already registered, rejecting", slave+1, url.str()); @@ -285,7 +285,7 @@ class CRegistryServer : public CSimpleInterface if (NotFound != connectedSlaves.find(sender)) { StringBuffer epStr; - PROGLOG("Same slave registered twice!! : %s", sender->endpoint().getUrlStr(epStr).str()); + PROGLOG("Same slave registered twice!! : %s", sender->endpoint().getEndpointHostText(epStr).str()); return false; } @@ -316,7 +316,7 @@ class CRegistryServer : public CSimpleInterface connectedSlaves.replace(sender.getLink(), pos); } StringBuffer epStr; - PROGLOG("Slave %u connected from %s", slaveNum, sender->endpoint().getUrlStr(epStr).str()); + PROGLOG("Slave %u connected from %s", slaveNum, sender->endpoint().getEndpointHostText(epStr).str()); --remaining; } assertex(slaves == connectedSlaves.ordinality()); @@ -396,12 +396,12 @@ class CRegistryServer : public CSimpleInterface break; s = ns+1; StringBuffer str; - PROGLOG("Slave %d (%s)", s, queryNodeGroup().queryNode(s).endpoint().getUrlStr(str.clear()).str()); + PROGLOG("Slave %d (%s)", s, queryNodeGroup().queryNode(s).endpoint().getEndpointHostText(str.clear()).str()); } throw MakeThorException(TE_AbortException, "Slaves failed to respond to cluster initialization"); } StringBuffer str; - PROGLOG("Registration confirmation from %s", queryNodeGroup().queryNode(sender).endpoint().getUrlStr(str).str()); + PROGLOG("Registration confirmation from %s", queryNodeGroup().queryNode(sender).endpoint().getEndpointHostText(str).str()); if (msg.length()) { Owned e = deserializeException(msg); @@ -543,7 +543,7 @@ bool checkClusterRelicateDAFS(IGroup &grp) SocketEndpoint ep(failures.item(i)); ep.port = 0; StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); FLLOG(MCoperatorError, thorJob, "VALIDATE FAILED(%d) %s : %s",failedcodes.item(i),ips.str(),failedmessages.item(i)); } PROGLOG("Cluster replicate nodes check completed in %dms",msTick()-start); @@ -984,7 +984,7 @@ int main( int argc, const char *argv[] ) thorJob.setJobID(thorJobId); setDefaultJobId(thorJobId); StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getUrlStr(thorEpStr).str()); + LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned numWorkersPerPod = 1; @@ -1019,7 +1019,7 @@ int main( int argc, const char *argv[] ) cloudJobName.appendf("%s-%s", workunit, graphName); StringBuffer myEp; - queryMyNode()->endpoint().getUrlStr(myEp); + getRemoteAccessibleHostText(myEp, queryMyNode()->endpoint()); workerNSInstalled = k8s::applyYaml("thorworker", workunit, cloudJobName, "networkpolicy", { }, false, true); if (workerNSInstalled) @@ -1033,7 +1033,7 @@ int main( int argc, const char *argv[] ) else { StringBuffer thorEpStr; - LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getUrlStr(thorEpStr).str()); + LOG(MCdebugProgress, thorJob, "ThorMaster version %d.%d, Started on %s", THOR_VERSION_MAJOR,THOR_VERSION_MINOR,thorEp.getEndpointHostText(thorEpStr).str()); LOG(MCdebugProgress, thorJob, "Thor name = %s, queue = %s, nodeGroup = %s",thorname,queueName.str(),nodeGroup.str()); unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_SLAVEPORTINC); unsigned slaveBasePort = globals->getPropInt("@slaveport", DEFAULT_THORSLAVEPORT); diff --git a/thorlcr/msort/tsortl.cpp b/thorlcr/msort/tsortl.cpp index 5cc42d1e544..d038c59e3bb 100644 --- a/thorlcr/msort/tsortl.cpp +++ b/thorlcr/msort/tsortl.cpp @@ -285,7 +285,7 @@ IRowStream *ConnectMergeRead(unsigned id, IThorRowInterfaces *rowif,SocketEndpoi TransferStreamHeader hdr(startrec, numrecs, id); #ifdef _FULL_TRACE StringBuffer s; - nodeaddr.getUrlStr(s); + nodeaddr.getEndpointHostText(s); LOG(MCthorDetailedDebugInfo, thorJob, "ConnectMergeRead(%d,%s,%x,%" RCPF "d,%" RCPF "u)",id,s.str(),(unsigned)(memsize_t)socket.get(),startrec,numrecs); #endif hdr.winrev(); diff --git a/thorlcr/msort/tsortm.cpp b/thorlcr/msort/tsortm.cpp index d2c78eca91f..4c346539ece 100644 --- a/thorlcr/msort/tsortm.cpp +++ b/thorlcr/msort/tsortm.cpp @@ -120,7 +120,7 @@ class CSortNode: public SortSlaveMP try { StringBuffer epstr; - ActPrintLog(activity, thorDetailedLogLevel, "Connect to %s:%d",endpoint.getIpText(epstr).str(),(unsigned)mpport); + ActPrintLog(activity, thorDetailedLogLevel, "Connect to %s:%d",endpoint.getHostText(epstr).str(),(unsigned)mpport); SocketEndpoint ep = endpoint; ep.port = mpport; Owned node = createINode(ep); @@ -268,7 +268,7 @@ class CSortMaster : public IThorSorterMaster, public CSimpleInterface CSortNode &slave = slaves.item(i); if (!slave.doConnect(i,slaves.ordinality())) { char url[100]; - slave.endpoint.getUrlStr(url,sizeof(url)); + slave.endpoint.getEndpointHostText(url,sizeof(url)); throw MakeActivityException(owner.activity,TE_CannotConnectToSlave,"CSortMaster::ConnectSlaves: Could not connect to %s",url); } } @@ -1235,7 +1235,7 @@ class CSortMaster : public IThorSorterMaster, public CSimpleInterface if (slave.scale!=mostspilt+1) { char url[100]; - slave.endpoint.getUrlStr(url,sizeof(url)); + slave.endpoint.getEndpointHostText(url,sizeof(url)); ActPrintLog(activity, "Gather - node %s spilled %d times to disk",url,slave.scale-1); } } @@ -1305,7 +1305,7 @@ class CSortMaster : public IThorSorterMaster, public CSimpleInterface { CSortNode &slave = slaves.item(i); char url[100]; - slave.endpoint.getUrlStr(url,sizeof(url)); + slave.endpoint.getEndpointHostText(url,sizeof(url)); ActPrintLog(activity, thorDetailedLogLevel, "Split point %d: %" RCPF "d rows on %s", i, tot[i], url); } } @@ -1331,7 +1331,7 @@ class CSortMaster : public IThorSorterMaster, public CSimpleInterface { CSortNode &slave = slaves.item(i); char url[100]; - slave.endpoint.getUrlStr(url,sizeof(url)); + slave.endpoint.getEndpointHostText(url,sizeof(url)); if (splitMapUpper) slave.MultiMergeBetween(total, numnodes*numnodes,splitMap,splitMapUpper,numnodes,endpoints.get()); else diff --git a/thorlcr/msort/tsortmp.cpp b/thorlcr/msort/tsortmp.cpp index 3280be77703..957ec55b238 100644 --- a/thorlcr/msort/tsortmp.cpp +++ b/thorlcr/msort/tsortmp.cpp @@ -301,7 +301,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag mbout.append(okout); #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getUrlStr(tmp1).str(),tag,mb.getReplyTag()); + LOG(MCthorDetailedDebugInfo, thorJob, ">SortSlaveMP::marshall(%d) got %d from %s tag %d replytag %d",(int)fn, mb.length(), mb.getSender().getEndpointHostText(tmp1).str(),tag,mb.getReplyTag()); #endif bool replydone = false; Owned err; @@ -531,7 +531,7 @@ bool SortSlaveMP::marshall(ISortSlaveMP &slave, ICommunicator* comm, mptag_t tag if (!replydone) { #ifdef FULLTRACE StringBuffer tmp1; - LOG(MCthorDetailedDebugInfo, thorJob, "reply(mbout); } diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index 3e3ed4a3479..e1f0c10cfb0 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -59,7 +59,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream { endpoint = targetep; char url[100]; - targetep.getUrlStr(url,sizeof(url)); + targetep.getEndpointHostText(url,sizeof(url)); LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: Stream(%u) %s, pos=%" RCPF "d len=%" RCPF "u",streamno,url,startrec,numrecs); SocketEndpoint mergeep = targetep; mergeep.port+=SOCKETSERVERINC; @@ -92,7 +92,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream { if (stream) { char url[100]; - endpoint.getUrlStr(url,sizeof(url)); + endpoint.getEndpointHostText(url,sizeof(url)); DBGLOG("SORT Merge READ: EOS via destructor for %s",url); stream->stop(); } @@ -107,7 +107,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream return row.getClear(); #ifdef _FULL_TRACE char url[100]; - endpoint.getUrlStr(url,sizeof(url)); + endpoint.getEndpointHostText(url,sizeof(url)); LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: EOS for %s",url); #endif eos(); @@ -120,7 +120,7 @@ class CMergeReadStream : public CSimpleInterface, public IRowStream if (stream) { #ifdef _FULL_TRACE char url[100]; - endpoint.getUrlStr(url,sizeof(url)); + endpoint.getEndpointHostText(url,sizeof(url)); LOG(MCthorDetailedDebugInfo, thorJob, "SORT Merge READ: stop for %s",url); #endif stream->stop(); @@ -315,7 +315,7 @@ protected: friend class CSortMerge; #if defined(_USE_OPENSSL) if (slave.queryTLS()) { - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); secureContextClients.setown(createSecureSocketContextSecret("local", ClientSocket)); } #endif @@ -492,7 +492,7 @@ protected: friend class CSortMerge; unsigned k = 0; for (i=0;iendpoint(); StringBuffer str("Node '"); - myEp.getUrlStr(str); + myEp.getEndpointHostText(str); str.append("' exception: ").append(errorMsg); Owned e = MakeStringException(errorCode, "%s", str.str()); CMessageBuffer msg; @@ -98,7 +98,7 @@ static bool RegisterSelf(SocketEndpoint &masterEp) { StringBuffer slfStr; StringBuffer masterStr; - LOG(MCdebugProgress, thorJob, "registering %s - master %s",slfEp.getUrlStr(slfStr).str(),masterEp.getUrlStr(masterStr).str()); + LOG(MCdebugProgress, thorJob, "registering %s - master %s",slfEp.getEndpointHostText(slfStr).str(),masterEp.getEndpointHostText(masterStr).str()); try { SocketEndpoint ep = masterEp; @@ -233,7 +233,7 @@ bool UnregisterSelf(IException *e) return false; StringBuffer slfStr; - slfEp.getUrlStr(slfStr); + slfEp.getEndpointHostText(slfStr); LOG(MCdebugProgress, thorJob, "Unregistering slave : %s", slfStr.str()); try { diff --git a/thorlcr/thorcodectx/thcodectx.cpp b/thorlcr/thorcodectx/thcodectx.cpp index 4d0fa79951b..797e1e0767a 100644 --- a/thorlcr/thorcodectx/thcodectx.cpp +++ b/thorlcr/thorcodectx/thcodectx.cpp @@ -84,7 +84,7 @@ char *CThorCodeContextBase::getDaliServers() first = false; else dali.append(','); - coven->query().endpoint().getUrlStr(dali); + coven->query().endpoint().getEndpointHostText(dali); } return dali.detach(); } diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp index df7463f4ec7..f2c6c2a2343 100644 --- a/thorlcr/thorutil/thormisc.cpp +++ b/thorlcr/thorutil/thormisc.cpp @@ -347,13 +347,13 @@ class DECL_EXCEPTION CThorException : public CSimpleInterface, implements IThorE { unsigned node = (unsigned)-slave; str.appendf("NODE #%d [", node); - queryNodeGroup().queryNode(node).endpoint().getUrlStr(str); + queryNodeGroup().queryNode(node).endpoint().getEndpointHostText(str); str.append("]: "); } else { str.appendf("SLAVE #%d [", slave); - queryClusterGroup().queryNode(slave).endpoint().getUrlStr(str); + queryClusterGroup().queryNode(slave).endpoint().getEndpointHostText(str); str.append("]: "); } } diff --git a/tools/backupnode/backupnode.cpp b/tools/backupnode/backupnode.cpp index bc59efaea9f..3f2406c1991 100644 --- a/tools/backupnode/backupnode.cpp +++ b/tools/backupnode/backupnode.cpp @@ -650,8 +650,8 @@ int main(int argc, const char *argv[]) if (!myip.ipequals(myipfromSlaves)) { StringBuffer ips1, ips2; - myipfromSlaves.getIpText(ips1); - myip.getIpText(ips2); + myipfromSlaves.getHostText(ips1); + myip.getHostText(ips2); printerr("IP address %d in slaves file %s does not match this machine %s", slaveNum, ips1.str(), ips2.str()); throw MakeStringException(-1, "IP address %d in slaves file %s does not match this machine %s", slaveNum, ips1.str(), ips2.str()); } diff --git a/tools/swapnode/swapnode.cpp b/tools/swapnode/swapnode.cpp index 77f6ed73046..1f2bdb3af35 100644 --- a/tools/swapnode/swapnode.cpp +++ b/tools/swapnode/swapnode.cpp @@ -279,7 +279,7 @@ int main(int argc, const char *argv[]) ForEachItemIn(e, epa) { StringBuffer ipStr; - epa.item(e).getIpText(ipStr); + epa.item(e).getHostText(ipStr); hosts.push_back(ipStr.str()); } } diff --git a/tools/swapnode/swapnodelib.cpp b/tools/swapnode/swapnodelib.cpp index f5eb6afc8ec..25cd2ff1875 100644 --- a/tools/swapnode/swapnodelib.cpp +++ b/tools/swapnode/swapnodelib.cpp @@ -281,11 +281,11 @@ class CSwapNode } SocketEndpoint ep(smtpserver.str(),25); StringBuffer sender("swapnode@"); - queryHostIP().getIpText(sender); + queryHostIP().getHostText(sender); // add tbd StringBuffer ips; StringArray warnings; - sendEmail(emailtarget.str(),subject,out.str(),ep.getIpText(ips).str(),ep.port,sender.str(),&warnings); + sendEmail(emailtarget.str(),subject,out.str(),ep.getHostText(ips).str(),ep.port,sender.str(),&warnings); ForEachItemIn(i,warnings) WARNLOG("SWAPNODE: %s",warnings.item(i)); } @@ -430,7 +430,7 @@ class CSwapNode SocketEndpoint ep(failures.item(i)); ep.port = 0; StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); int r = (int)grp->rank(ep); if (r<0) { // shouldn't occur ERRLOG("SWAPNODE node %s not found in group %s",ips.str(),groupName.get()); @@ -601,7 +601,7 @@ class CAutoSwapNode : public CSwapNode SocketEndpoint ep(badepa.item(i1)); ep.port = 0; // should be no ports in group StringBuffer ips; - ep.getIpText(ips); + ep.getHostText(ips); xpath.clear().appendf("BadNode[@netAddress=\"%s\"]",ips.str()); IPropertyTree *bnt = info->queryPropTree(xpath.str()); if (!bnt) { @@ -622,7 +622,7 @@ class CAutoSwapNode : public CSwapNode if ((r==(r1+1)%grp->ordinality())|| (r1==(r+1)%grp->ordinality())) { StringBuffer ips1; - ep1.getIpText(ips1); + ep1.getHostText(ips1); ERRLOG("SWAPNODE adjacent nodes %d (%s) and %d (%s) are bad!",r+1,ips.str(),r1+1,ips1.str()); abort = true; } @@ -660,7 +660,7 @@ class CAutoSwapNode : public CSwapNode else if ((badr==(r1+1)%grp->ordinality())|| (r1==(badr+1)%grp->ordinality())) { StringBuffer bs; - ERRLOG("SWAPNODE adjacent node to bad node %d (%s), %d (%s) was swapped on %s (too recent) !",badr+1,badep.getIpText(bs).str(),r1+1,ips,dt1s); + ERRLOG("SWAPNODE adjacent node to bad node %d (%s), %d (%s) was swapped on %s (too recent) !",badr+1,badep.getHostText(bs).str(),r1+1,ips,dt1s); abort = true; } } @@ -682,10 +682,10 @@ class CAutoSwapNode : public CSwapNode spareGroup->getSocketEndpoints(spareepa); ForEachItemIn(i3,badepa) { StringBuffer from; - badepa.item(i3).getIpText(from); + badepa.item(i3).getHostText(from); if (i3rank(spareEp); dbgassertex(RANK_NULL != r); spareGroup.setown(spareGroup->remove(r)); diff --git a/vcpkg b/vcpkg index 4c3f6e34fe1..19a480bbf1f 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit 4c3f6e34fe1e206e327000888442a3b65e82c3ec +Subproject commit 19a480bbf1f8a8f29f03fb5c719b560fd7ed7f65