From a5ba772ed86de71f2567ebe15700a4fb0d8308fb Mon Sep 17 00:00:00 2001 From: Anthony Fishbeck Date: Wed, 26 Jul 2023 15:59:31 -0400 Subject: [PATCH] HPCC-30131 Cloud: Support HPCC Remote Trust via shared cert authority Signed-off-by: Anthony Fishbeck --- common/thorhelper/thorsoapcall.cpp | 24 +- esp/clients/wsdfuaccess/wsdfuaccess.cpp | 2 +- esp/services/ws_dfu/ws_dfuService.cpp | 4 +- esp/services/ws_ecl/ws_ecl_service.cpp | 2 +- fs/dafilesrv/dafilesrv.cpp | 2 +- fs/dafsclient/rmtclient.cpp | 2 +- fs/dafsserver/dafsserver.cpp | 2 +- helm/examples/certmanager/README-vault-pki.md | 361 ------------------ .../certmanager/values-vault-pki.yaml | 33 -- .../vault-pki-remote/README-vault-pki.md | 237 ++++++++++++ .../vault-pki-remote/local-ca-req.cfg | 18 + .../examples/vault-pki-remote/remote_echo.ecl | 59 +++ helm/examples/vault-pki-remote/roxie_echo.ecl | 24 ++ .../vault-pki-remote/signing-ca-req.cfg | 18 + .../vault-pki-remote/values-hpcc1.yaml | 98 +++++ .../vault-pki-remote/values-hpcc2.yaml | 98 +++++ helm/hpcc/templates/_helpers.tpl | 149 ++++++-- helm/hpcc/templates/_warnings.tpl | 2 +- helm/hpcc/templates/dafilesrv.yaml | 2 +- helm/hpcc/templates/eclagent.yaml | 12 +- helm/hpcc/templates/eclccserver.yaml | 12 +- helm/hpcc/templates/esp.yaml | 16 +- helm/hpcc/templates/localroxie.yaml | 6 +- helm/hpcc/templates/roxie.yaml | 17 +- helm/hpcc/templates/thor.yaml | 30 +- helm/hpcc/values.schema.json | 21 + helm/hpcc/values.yaml | 26 +- roxie/ccd/ccdmain.cpp | 73 ++-- roxie/ccd/ccdprotocol.cpp | 27 +- roxie/ccd/hpccprotocol.hpp | 2 +- system/jlib/jsecrets.cpp | 34 +- system/jlib/jsecrets.hpp | 7 +- system/jlib/jsmartsock.cpp | 3 +- system/jlib/jsmartsock.ipp | 2 + system/mp/mpcomm.cpp | 2 +- system/security/securesocket/securesocket.cpp | 18 +- system/security/securesocket/securesocket.hpp | 2 +- thorlcr/msort/tsorts1.cpp | 2 +- 38 files changed, 890 insertions(+), 559 deletions(-) delete mode 100644 helm/examples/certmanager/README-vault-pki.md delete mode 100644 helm/examples/certmanager/values-vault-pki.yaml create mode 100644 helm/examples/vault-pki-remote/README-vault-pki.md create mode 100644 helm/examples/vault-pki-remote/local-ca-req.cfg create mode 100644 helm/examples/vault-pki-remote/remote_echo.ecl create mode 100644 helm/examples/vault-pki-remote/roxie_echo.ecl create mode 100644 helm/examples/vault-pki-remote/signing-ca-req.cfg create mode 100644 helm/examples/vault-pki-remote/values-hpcc1.yaml create mode 100644 helm/examples/vault-pki-remote/values-hpcc2.yaml diff --git a/common/thorhelper/thorsoapcall.cpp b/common/thorhelper/thorsoapcall.cpp index 085f0e18d91..6d1fa2db1d0 100644 --- a/common/thorhelper/thorsoapcall.cpp +++ b/common/thorhelper/thorsoapcall.cpp @@ -876,6 +876,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface static CriticalSection secureContextCrit; static Owned tlsSecureContext; static Owned localMtlsSecureContext; + static Owned remoteMtlsSecureContext; Owned customSecureContext; @@ -883,7 +884,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface bool complete; std::atomic_bool timeLimitExceeded{false}; bool customClientCert = false; - bool localClientCert = false; + StringAttr clientCertIssuer; IRoxieAbortMonitor * roxieAbortMonitor; protected: @@ -1021,9 +1022,14 @@ class CWSCHelper : implements IWSCHelper, public CInterface throw MakeStringException(0, "%sCALL specified no URLs",wscType == STsoap ? "SOAP" : "HTTP"); if (0==strncmp(hosts, "mtls:", 5)) { - localClientCert = true; + clientCertIssuer.set("local"); hosts += 5; } + else if (0==strncmp(hosts, "remote-mtls:", 12)) + { + clientCertIssuer.set("remote"); + hosts += 12; + } if (0==strncmp(hosts, "secret:", 7)) { const char *finger = hosts+7; @@ -1184,8 +1190,8 @@ class CWSCHelper : implements IWSCHelper, public CInterface { if (clientCert != NULL) ownedSC.setown(createSecureSocketContextEx(clientCert->certificate, clientCert->privateKey, clientCert->passphrase, ClientSocket)); - else if (localClientCert) - ownedSC.setown(createSecureSocketContextSecret("local", ClientSocket)); + else if (clientCertIssuer.length()) + ownedSC.setown(createSecureSocketContextSecret(clientCertIssuer.str(), ClientSocket)); else ownedSC.setown(createSecureSocketContext(ClientSocket)); } @@ -1194,8 +1200,13 @@ class CWSCHelper : implements IWSCHelper, public CInterface ISecureSocketContext *ensureStaticSecureContext() { CriticalBlock b(secureContextCrit); - if (localClientCert) - return ensureSecureContext(localMtlsSecureContext); + if (clientCertIssuer.length()) + { + if (strieq(clientCertIssuer.str(), "local")) + return ensureSecureContext(localMtlsSecureContext); + if (strieq(clientCertIssuer.str(), "remote")) + return ensureSecureContext(remoteMtlsSecureContext); + } return ensureSecureContext(tlsSecureContext); } ISecureSocket *createSecureSocket(ISocket *sock, const char *fqdn = nullptr) @@ -1333,6 +1344,7 @@ class CWSCHelper : implements IWSCHelper, public CInterface CriticalSection CWSCHelper::secureContextCrit; Owned CWSCHelper::tlsSecureContext; // created on first use Owned CWSCHelper::localMtlsSecureContext; // created on first use +Owned CWSCHelper::remoteMtlsSecureContext; // created on first use //================================================================================================= diff --git a/esp/clients/wsdfuaccess/wsdfuaccess.cpp b/esp/clients/wsdfuaccess/wsdfuaccess.cpp index 8a585bc7a75..7999197574d 100644 --- a/esp/clients/wsdfuaccess/wsdfuaccess.cpp +++ b/esp/clients/wsdfuaccess/wsdfuaccess.cpp @@ -543,7 +543,7 @@ StringBuffer &encodeDFUFileMeta(StringBuffer &metaInfoBlob, IPropertyTree *metaI * If the size of this initial request was ever a concern, we could consider other ways to ensure a one-off * delivery of this esp public signing cert. to dafilesrv, e.g. by dafilesrv reaching out to esp to request it. */ - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "encodeDFUFileMeta: No '%s' MTLS certificate detected.", keyPairName); privateKeyFName = info->queryProp("privatekey"); diff --git a/esp/services/ws_dfu/ws_dfuService.cpp b/esp/services/ws_dfu/ws_dfuService.cpp index 4dcfd29b947..6eac98359d2 100644 --- a/esp/services/ws_dfu/ws_dfuService.cpp +++ b/esp/services/ws_dfu/ws_dfuService.cpp @@ -6112,7 +6112,7 @@ void CWsDfuEx::dFUFileAccessCommon(IEspContext &context, const CDfsLogicalFileNa StringBuffer dafilesrvHost; #ifdef _CONTAINERIZED keyPairName.set("signing"); - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "dFUFileAccessCommon: file signing certificate ('%s') not defined in configuration.", keyPairName.str()); @@ -6489,7 +6489,7 @@ bool CWsDfuEx::onDFUFileCreateV2(IEspContext &context, IEspDFUFileCreateV2Reques #ifdef _CONTAINERIZED keyPairName.set("signing"); - IPropertyTree *info = queryTlsSecretInfo(keyPairName); + Owned info = getIssuerTlsServerConfig(keyPairName); if (!info) throw makeStringExceptionV(-1, "onDFUFileCreateV2: file signing certificate ('%s' ) not defined in configuration.", keyPairName.str()); diff --git a/esp/services/ws_ecl/ws_ecl_service.cpp b/esp/services/ws_ecl/ws_ecl_service.cpp index 12317eb08a6..fcc042e8f82 100644 --- a/esp/services/ws_ecl/ws_ecl_service.cpp +++ b/esp/services/ws_ecl/ws_ecl_service.cpp @@ -2082,8 +2082,8 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri throw MakeStringException(-1, "roxie target cluster not mapped: %s", target); ep = conn->nextEndpoint(); - Owned httpctx = getHttpClientContext(); WsEclSocketFactory *roxieConn = static_cast(conn); + Owned httpctx = getHttpClientSecretContext(roxieConn->queryTlsIssuer()); StringBuffer url(roxieConn->isTlsService() ? "https://" : "http://"); ep.getIpText(url).append(':').append(ep.port ? ep.port : 9876).append('/'); if (roxieConn->includeTargetInURL) diff --git a/fs/dafilesrv/dafilesrv.cpp b/fs/dafilesrv/dafilesrv.cpp index efbf863e5a6..19cdf0becb8 100644 --- a/fs/dafilesrv/dafilesrv.cpp +++ b/fs/dafilesrv/dafilesrv.cpp @@ -395,7 +395,7 @@ int main(int argc, const char* argv[]) // Use the "public" certificate issuer, unless it's visibility is "cluster" (meaning internal only) const char *visibility = getComponentConfigSP()->queryProp("service/@visibility"); const char *certScope = strsame("cluster", visibility) ? "local" : "public"; - IPropertyTree *info = queryTlsSecretInfo(certScope); + Owned info = getIssuerTlsServerConfig(certScope); connectMethod = info ? SSLOnly : SSLNone; // NB: connectMethod will direct the CRemoteFileServer on accept to create a secure socket based on the same issuer certificates diff --git a/fs/dafsclient/rmtclient.cpp b/fs/dafsclient/rmtclient.cpp index 1a0b2123c55..b5c216b968c 100644 --- a/fs/dafsclient/rmtclient.cpp +++ b/fs/dafsclient/rmtclient.cpp @@ -156,7 +156,7 @@ static ISecureSocket *createSecureSocket(ISocket *sock, const char *issuer) auto it = secureCtxClientIssuerMap.find(issuer); if (it == secureCtxClientIssuerMap.end()) { - IPropertyTree *info = queryTlsSecretInfo(issuer); + Owned info = getIssuerTlsServerConfig(issuer); if (!info) throw makeStringExceptionV(-1, "createSecureSocket() : missing MTLS configuration for issuer: %s", issuer); secureContext.setown(createSecureSocketContextEx2(info, ClientSocket)); diff --git a/fs/dafsserver/dafsserver.cpp b/fs/dafsserver/dafsserver.cpp index 3ce29430a25..94860818379 100644 --- a/fs/dafsserver/dafsserver.cpp +++ b/fs/dafsserver/dafsserver.cpp @@ -133,7 +133,7 @@ static ISecureSocket *createSecureSocket(ISocket *sock, bool disableClientCertVe */ const char *certScope = strsame("cluster", getComponentConfigSP()->queryProp("service/@visibility")) ? "local" : "public"; - IPropertyTree *info = queryTlsSecretInfo(certScope); + Owned info = getIssuerTlsServerConfig(certScope); if (!info) throw makeStringException(-1, "createSecureSocket() : missing MTLS configuration"); Owned cloneInfo; diff --git a/helm/examples/certmanager/README-vault-pki.md b/helm/examples/certmanager/README-vault-pki.md deleted file mode 100644 index 78837f751f6..00000000000 --- a/helm/examples/certmanager/README-vault-pki.md +++ /dev/null @@ -1,361 +0,0 @@ -# Install Hashicorp Vault - -See also https://learn.hashicorp.com/tutorials/vault/kubernetes-cert-manager for a more Vault centric tutorial on setting up cert-manager with vault. - -## Add hashicorp to you helm repo -```bash -helm repo add hashicorp https://helm.releases.hashicorp.com -``` - -## Helm install hashicorp vault - -Disable the vault sidecar injector by setting "injector.enabled=false". - -```bash -helm install vault hashicorp/vault --set "injector.enabled=false" -``` - -Check the pods: -```bash -kubectl get pods -``` - -Vault pods should be running, but not ready - -```bash -$ kubectl get pods -NAME READY STATUS RESTARTS AGE -vault-0 0/1 Running 0 6s -``` - -## Initialize and unseal the vault - -Initialize Vault with one key share and one key threshold. Saving off the output in json format so -we can utilize the unseal key and root token later. - -```bash -kubectl exec vault-0 -- vault operator init -key-shares=1 -key-threshold=1 -format=json > init-keys.json -``` - -View the unseal key found in init-keys.json. - -```bash -cat init-keys.json | jq -r ".unseal_keys_b64[]" -``` - -Create an environment variable holding the unseal key: - -```bash -VAULT_UNSEAL_KEY=$(cat init-keys.json | jq -r ".unseal_keys_b64[]") -``` - -Unseal Vault running on the vault-0 pod with the $VAULT_UNSEAL_KEY. - -```bash -kubectl exec vault-0 -- vault operator unseal $VAULT_UNSEAL_KEY -``` - -Check the pods: -```bash -kubectl get pods -``` - -Vault pods should now be running and ready. - -## Configure the Vault PKI secrets engine (certificate authority) - -View the vault root token: -```bash -cat init-keys.json | jq -r ".root_token" -``` - -Create a variable named VAULT_ROOT_TOKEN to capture the root token. -```bash -VAULT_ROOT_TOKEN=$(cat init-keys.json | jq -r ".root_token") -``` - -Login to Vault running on the vault-0 pod with the $VAULT_ROOT_TOKEN. -```bash -kubectl exec vault-0 -- vault login $VAULT_ROOT_TOKEN -``` - -Start an interactive shell session on the vault-0 pod. -```bash -kubectl exec --stdin=true --tty=true vault-0 -- /bin/sh -``` -We are now working from the vault-0 pod. You should see a prompt, something like: - -```bash -/ $ -``` - -Enable the PKI secrets engine at its default path. -```bash -vault secrets enable pki -``` - -Configure the max lease time-to-live (TTL) to 8760h. -```bash -vault secrets tune -max-lease-ttl=8760h pki -``` - -# Vault CA key pair - -Vault can accept an existing key pair, or it can generate its own self-signed root. In general, they recommend maintaining your root CA outside of Vault and providing Vault a signed intermediate CA, but for this demo we will keep it simple and generate a self signed root certificate. - -Generate a self-signed certificate valid for 8760h. -```bash -vault write pki/root/generate/internal common_name=example.com ttl=8760h -``` - -Configure the PKI secrets engine certificate issuing and certificate revocation list (CRL) endpoints to use the Vault service in the default namespace. -```bash -vault write pki/config/urls issuing_certificates="http://vault.default:8200/v1/pki/ca" crl_distribution_points="http://vault.default:8200/v1/pki/crl" -``` - -For our local MTLS certificates we will use our kubernetes namespace as our domain name. This will allow us to recongize where these components reside. -For our public TLS certificates for this demo we will use myhpcc.com as our domain. - -Configure a role named hpccnamespace that enables the creation of certificates hpccnamespace domain with any subdomains. - -```bash -vault write pki/roles/hpcclocal key_type=any allowed_domains=default allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72h -``` - -Configure a role named myhpcc-dot-com that enables the creation of certificates myhpcc.com domain with any subdomains. - -```bash -vault write pki/roles/myhpcc-dot-com allowed_domains=myhpcc.com allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72h -``` - -Create a policy named pki that enables read access to the PKI secrets engine paths. - -```bash -vault policy write pki - < - - GET/WsSmc/HttpEchoname=doe,joe&number=1
Accept-Encoding: gzip, deflate
Accept: text/xml
 - - -``` diff --git a/helm/examples/certmanager/values-vault-pki.yaml b/helm/examples/certmanager/values-vault-pki.yaml deleted file mode 100644 index 0f44fdfcd2d..00000000000 --- a/helm/examples/certmanager/values-vault-pki.yaml +++ /dev/null @@ -1,33 +0,0 @@ -certificates: - issuers: - local: - spec: - # delete any alternative spec config, and then add vault - ca: null - selfSigned: null - vault: - server: http://vault.default:8200 - path: pki/sign/hpcclocal - auth: - kubernetes: - mountPath: /v1/auth/kubernetes - role: issuer - secretRef: - name: tbd # requires service account secret, set from command line - key: token - public: - domain: myhpcc.com - spec: - # delete any alternative spec config, and then add vault - ca: null - selfSigned: null - vault: - server: http://vault.default:8200 - path: pki/sign/myhpcc-dot-com - auth: - kubernetes: - mountPath: /v1/auth/kubernetes - role: issuer - secretRef: - name: tbd # requires service account secret, set from command line - key: token diff --git a/helm/examples/vault-pki-remote/README-vault-pki.md b/helm/examples/vault-pki-remote/README-vault-pki.md new file mode 100644 index 00000000000..c8c74e1e35c --- /dev/null +++ b/helm/examples/vault-pki-remote/README-vault-pki.md @@ -0,0 +1,237 @@ +# Using a Hashicorp Vault PKI Certificate Authority to establish trust between two HPCC environments + +This walkthough demonstrates using a single Hashicorp Vault PKI Certificate quthority to establish trust between two or more HPCC environments. + +In the case of this example each HPCC environment is in a separate kubernetes namespace. + +## Install hashicorp vault service in dev mode: + +This is for development only, never deploy this way in production. +Deploying in dev mode sets up an in memory kv store that won't persist secret values across restart, and the vault will automatically be unsealed. + +In dev mode the default root token is simply the string "root". + +Add Hashicorp helm repo: + +```bash +helm repo add hashicorp https://helm.releases.hashicorp.com +``` + +Update Helm repos. + +```bash +helm repo update +``` + +Install vault server. + +Note that a recent change to the developer mode vault means that you have to set the VAULT_DEV_LISTEN_ADDRESS environment variable as shown in order to access the vault service from an external pod. + +```bash +helm install vault hashicorp/vault --set "injector.enabled=false" --set "server.dev.enabled=true" --set 'server.extraEnvironmentVars.VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' --namespace vaultns --create-namespace +``` + +Check the pods: +```bash +kubectl get pods -n vaultns +``` + +Vault pods should now be running and ready. + + +## Setting up vault + +Tell the vault command line application the server location (dev mode is http, default location is https) + +```bash +export VAULT_ADDR=http://127.0.0.1:8200 +``` + +Export an environment variable for the vault CLI to authenticate with the Vault server. Because we installed dev mode, the vault token is 'root'. + +```bash +export VAULT_TOKEN=root +``` + +In a separate terminal window start vault port forwarding. + +```bash +kubectl port-forward vault-0 8200:8200 -n vaultns +``` + +Login to the vault command line using the vault root token (development mode defaults to "root"): + +```bash +vault login root +``` + +## Enable the PKI secrets engine at its default path. +```bash +vault secrets enable pki +``` + +Configure the max lease time-to-live (TTL) to 8760h. +```bash +vault secrets tune -max-lease-ttl=87600h pki +``` + +Generate the hpcc remote issuer CA, give it an issuer name. + +```bash +vault write -field=certificate pki/root/generate/internal common_name="hpcc-issuer" issuer_name="hpcc-remote-issuer" ttl=87600h +``` + +Configure the PKI secrets engine certificate issuing and certificate revocation list (CRL) endpoints to use the Vault service in the "vaultns" namespace. + +If you installed vault into a different namespace update the urls, replacing "vaultns" with the namespace used. + +```bash +vault write pki/config/urls issuing_certificates="http://vault.vaultns:8200/v1/pki/ca" crl_distribution_points="http://vault.vaultns:8200/v1/pki/crl" +``` + +For our local MTLS certificates we will use our kubernetes namespace as our domain name. This will allow us to recongize where these components reside. +For our public TLS certificates for this demo we will use myhpcc.com as our domain. + +Configure a role named hpccnamespace that enables the creation of certificates hpccnamespace domain with any subdomains. + +```bash +vault write pki/roles/hpccremote key_type=any allowed_domains="hpcc1,hpcc2" allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72 +``` + +Create a policy named pki that enables read access to the PKI secrets engine paths. + +```bash +vault policy write hpcc-remote-pki - < protocolPlugin = loadHpccProtocolPlugin(protocolCtx, NULL); - Owned roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(myNode.getIpAddress(), 0, 1, false), 0, 0, NULL); + Owned roxieServer = protocolPlugin->createListener("runOnce", createRoxieProtocolMsgSink(myNode.getIpAddress(), 0, 1, false), 0, 0, nullptr, nullptr, nullptr, nullptr, nullptr); try { const char *format = topology->queryProp("@format"); @@ -1502,7 +1502,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) { roxiePort = port; if (roxieFarm.getPropBool("@tls")) - roxiePortTlsClientConfig = createTlsClientSecretInfo(roxieFarm.queryProp("@issuer"), !roxieFarm.getPropBool("@public"), roxieFarm.getPropBool("@selfSigned")); + roxiePortTlsClientConfig = createIssuerTlsClientConfig(roxieFarm.queryProp("@issuer"), roxieFarm.getPropBool("@selfSigned")); debugEndpoint.set(roxiePort, ip); } bool suspended = roxieFarm.getPropBool("@suspended", false); @@ -1514,40 +1514,49 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) StringBuffer certFileName; StringBuffer keyFileName; StringBuffer passPhraseStr; + Owned tlsConfig; if (serviceTLS) { protocol = "ssl"; #ifdef _USE_OPENSSL - #ifdef _CONTAINERIZED - const char *certIssuer = roxieFarm.getPropBool("@public", true) ? "public" : "local"; - certFileName.setf("/opt/HPCCSystems/secrets/certificates/%s/tls.crt", certIssuer); - keyFileName.setf("/opt/HPCCSystems/secrets/certificates/%s/tls.key", certIssuer); - #else - const char *passPhrase = roxieFarm.queryProp("@passphrase"); - if (!isEmptyString(passPhrase)) - decrypt(passPhraseStr, passPhrase); - - const char *certFile = roxieFarm.queryProp("@certificateFileName"); - if (!certFile) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port); - if (isAbsolutePath(certFile)) - certFileName.append(certFile); - else - certFileName.append(codeDirectory.str()).append(certFile); - - const char *keyFile = roxieFarm.queryProp("@privateKeyFileName"); - if (!keyFile) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port); - if (isAbsolutePath(keyFile)) - keyFileName.append(keyFile); + if (isContainerized()) + { + const char *certIssuer = roxieFarm.queryProp("@issuer"); + if (isEmptyString(certIssuer)) + certIssuer = roxieFarm.getPropBool("@public", true) ? "public" : "local"; + tlsConfig.setown(getIssuerTlsServerConfigWithTrustedPeers(certIssuer, roxieFarm.queryProp("trusted_peers"))); + if (!tlsConfig) + throw MakeStringException(ROXIE_FILE_ERROR, "TLS secret for issuer %s not found", certIssuer); + DBGLOG("Roxie service, port(%d) TLS issuer (%s)", port, certIssuer); + } else - keyFileName.append(codeDirectory.str()).append(keyFile); - #endif - if (!checkFileExists(certFileName.str())) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str()); - - if (!checkFileExists(keyFileName.str())) - throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str()); + { + const char *passPhrase = roxieFarm.queryProp("@passphrase"); + if (!isEmptyString(passPhrase)) + decrypt(passPhraseStr, passPhrase); + + const char *certFile = roxieFarm.queryProp("@certificateFileName"); + if (!certFile) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFileName tag", port); + if (isAbsolutePath(certFile)) + certFileName.append(certFile); + else + certFileName.append(codeDirectory.str()).append(certFile); + + const char *keyFile = roxieFarm.queryProp("@privateKeyFileName"); + if (!keyFile) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFileName tag", port); + if (isAbsolutePath(keyFile)) + keyFileName.append(keyFile); + else + keyFileName.append(codeDirectory.str()).append(keyFile); + + if (!checkFileExists(certFileName.str())) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing certificateFile (%s)", port, certFileName.str()); + + if (!checkFileExists(keyFileName.str())) + throw MakeStringException(ROXIE_FILE_ERROR, "Roxie SSL Farm Listener on port %d missing privateKeyFile (%s)", port, keyFileName.str()); + } #else OWARNLOG("Skipping Roxie SSL Farm Listener on port %d : OpenSSL disabled in build", port); @@ -1558,7 +1567,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml) const char *config = roxieFarm.queryProp("@config"); // NB: leaks - until we fix bug in ensureProtocolPlugin() whereby some paths return a linked object and others do not IHpccProtocolPlugin *protocolPlugin = ensureProtocolPlugin(*protocolCtx, soname); - roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, certFileName.str(), keyFileName.str(), passPhraseStr.str())); + roxieServer.setown(protocolPlugin->createListener(protocol ? protocol : "native", createRoxieProtocolMsgSink(ip, port, numThreads, suspended), port, listenQueue, config, tlsConfig, certFileName, keyFileName, passPhraseStr)); } else roxieServer.setown(createRoxieWorkUnitListener(numThreads, suspended)); diff --git a/roxie/ccd/ccdprotocol.cpp b/roxie/ccd/ccdprotocol.cpp index f0586bbf3eb..0dbfd6b233a 100644 --- a/roxie/ccd/ccdprotocol.cpp +++ b/roxie/ccd/ccdprotocol.cpp @@ -29,7 +29,7 @@ //================================================================================================================================ -IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile, const char *keyFile, const char *passPhrase); +IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase); class CHpccProtocolPlugin : implements IHpccProtocolPlugin, public CInterface { @@ -60,9 +60,9 @@ class CHpccProtocolPlugin : implements IHpccProtocolPlugin, public CInterface maxHttpConnectionRequests = ctx.ctxGetPropInt("@maxHttpConnectionRequests", 0); maxHttpKeepAliveWait = ctx.ctxGetPropInt("@maxHttpKeepAliveWait", 5000); // In milliseconds } - IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr) + IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase) { - return createProtocolListener(protocol, sink, port, listenQueue, certFile, keyFile, passPhrase); + return createProtocolListener(protocol, sink, port, listenQueue, tlsConfig, certFile, keyFile, passPhrase); } public: StringArray targetNames; @@ -231,7 +231,7 @@ class ProtocolSocketListener : public ProtocolListener bool isSSL = false; public: - ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue, const char *_protocol, const char *_certFile, const char *_keyFile, const char *_passPhrase) + ProtocolSocketListener(IHpccProtocolMsgSink *_sink, unsigned _port, unsigned _listenQueue, const char *_protocol, const IPropertyTree *_tlsConfig, const char *_certFile, const char *_keyFile, const char *_passPhrase) : ProtocolListener(_sink) { port = _port; @@ -242,9 +242,15 @@ class ProtocolSocketListener : public ProtocolListener keyFile.set(_keyFile); passPhrase.set(_passPhrase); isSSL = streq(protocol.str(), "ssl"); + #ifdef _USE_OPENSSL if (isSSL) - secureContext.setown(createSecureSocketContextEx(certFile.get(), keyFile.get(), passPhrase.get(), ServerSocket)); + { + if (_tlsConfig) + secureContext.setown(createSecureSocketContextEx2(_tlsConfig, ServerSocket)); + else + secureContext.setown(createSecureSocketContextEx(certFile.get(), keyFile.get(), passPhrase.get(), ServerSocket)); + } #endif } @@ -2222,11 +2228,16 @@ void ProtocolSocketListener::runOnce(const char *query) p->runOnce(query); } -IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr) +IHpccProtocolListener *createProtocolListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase) { if (traceLevel) - DBGLOG("Creating Roxie socket listener, protocol %s, pool size %d, listen queue %d%s", protocol, sink->getPoolSize(), listenQueue, sink->getIsSuspended() ? " SUSPENDED":""); - return new ProtocolSocketListener(sink, port, listenQueue, protocol, certFile, keyFile, passPhrase); + { + const char *certIssuer = "none"; + if (tlsConfig && tlsConfig->hasProp("@issuer")) + certIssuer = tlsConfig->queryProp("@issuer"); + DBGLOG("Creating Roxie socket listener, protocol %s, issuer=%s, pool size %d, listen queue %d%s", protocol, certIssuer, sink->getPoolSize(), listenQueue, sink->getIsSuspended() ? " SUSPENDED":""); + } + return new ProtocolSocketListener(sink, port, listenQueue, protocol, tlsConfig, certFile, keyFile, passPhrase); } extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *_limiterFactory) diff --git a/roxie/ccd/hpccprotocol.hpp b/roxie/ccd/hpccprotocol.hpp index cd09e589556..60cdb842bc7 100644 --- a/roxie/ccd/hpccprotocol.hpp +++ b/roxie/ccd/hpccprotocol.hpp @@ -135,7 +135,7 @@ interface IActiveQueryLimiterFactory : extends IInterface interface IHpccProtocolPlugin : extends IInterface { - virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const char *certFile=nullptr, const char *keyFile=nullptr, const char *passPhrase=nullptr)=0; + virtual IHpccProtocolListener *createListener(const char *protocol, IHpccProtocolMsgSink *sink, unsigned port, unsigned listenQueue, const char *config, const IPropertyTree *tlsConfig, const char *certFile, const char *keyFile, const char *passPhrase)=0; }; extern IHpccProtocolPlugin *loadHpccProtocolPlugin(IHpccProtocolPluginContext *ctx, IActiveQueryLimiterFactory *limiterFactory); diff --git a/system/jlib/jsecrets.cpp b/system/jlib/jsecrets.cpp index 8a99f467dc7..23183c83a4c 100644 --- a/system/jlib/jsecrets.cpp +++ b/system/jlib/jsecrets.cpp @@ -1085,7 +1085,7 @@ jlib_decl bool containsEmbeddedKey(const char *certificate) return false; } -IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert) +IPropertyTree *createIssuerTlsClientConfig(const char *issuer, bool acceptSelfSigned, bool addCACert) { if (isEmptyString(issuer)) return nullptr; @@ -1096,7 +1096,7 @@ IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool a Owned info = createPTree(); - if (mutual) + if (strieq(issuer, "remote")||strieq(issuer, "local")) { filepath.set(secretpath).append("tls.crt"); if (!checkFileExists(filepath)) @@ -1126,7 +1126,7 @@ IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool a return info.getClear(); } -IPropertyTree *queryTlsSecretInfo(const char *name) +IPropertyTree *getIssuerTlsServerConfig(const char *name) { if (isEmptyString(name)) return nullptr; @@ -1134,9 +1134,9 @@ IPropertyTree *queryTlsSecretInfo(const char *name) validateSecretName(name); CriticalBlock block(mtlsInfoCacheCS); - IPropertyTree *info = mtlsInfoCache->queryPropTree(name); + Owned info = mtlsInfoCache->getPropTree(name); if (info) - return info; + return info.getClear(); StringBuffer filepath; StringBuffer secretpath; @@ -1147,7 +1147,8 @@ IPropertyTree *queryTlsSecretInfo(const char *name) if (!checkFileExists(filepath)) return nullptr; - info = mtlsInfoCache->setPropTree(name); + info.set(mtlsInfoCache->setPropTree(name)); + info->setProp("@issuer", name); info->setProp("certificate", filepath.str()); filepath.set(secretpath).append("tls.key"); if (checkFileExists(filepath)) @@ -1162,13 +1163,28 @@ IPropertyTree *queryTlsSecretInfo(const char *name) if (ca) ca->setProp("@path", filepath.str()); } - // TLS TODO: do we want to always require verify, even if no ca ? - verify->setPropBool("@enable", true); + //For now only the "public" issuer implies client certificates are not required + verify->setPropBool("@enable", !strieq(name, "public")); verify->setPropBool("@address_match", false); verify->setPropBool("@accept_selfsigned", false); verify->setProp("trusted_peers", "anyone"); } - return info; + return info.getClear(); +} + +IPropertyTree *getIssuerTlsServerConfigWithTrustedPeers(const char *issuer, const char *trusted_peers) +{ + Owned issuerConfig = getIssuerTlsServerConfig(issuer); + if (!issuerConfig || isEmptyString(trusted_peers)) + return issuerConfig.getClear(); + //TBD: might cache in the future, but needs thought, lookup must include trusted_peers, but will there be cases where trusted_peers can change dynamically? + Owned tlsConfig = createPTreeFromIPT(issuerConfig); + if (!tlsConfig) + return nullptr; + + IPropertyTree *verify = ensurePTree(tlsConfig, "verify"); + verify->setProp("trusted_peers", trusted_peers); + return tlsConfig.getClear(); } enum UseMTLS { UNINIT, DISABLED, ENABLED }; diff --git a/system/jlib/jsecrets.hpp b/system/jlib/jsecrets.hpp index c473cdc0c84..4638fa6cf30 100644 --- a/system/jlib/jsecrets.hpp +++ b/system/jlib/jsecrets.hpp @@ -38,8 +38,11 @@ extern jlib_decl const MemoryAttr &getSecretUdpKey(bool required); extern jlib_decl bool containsEmbeddedKey(const char *certificate); -extern jlib_decl IPropertyTree *queryTlsSecretInfo(const char *issuer); -extern jlib_decl IPropertyTree *createTlsClientSecretInfo(const char *issuer, bool mutual, bool acceptSelfSigned, bool addCACert=true); +//getIssuerTlsServerConfig must return owned because the internal cache could be updated internally and the return will become invalid, so must be linked +extern jlib_decl IPropertyTree *getIssuerTlsServerConfig(const char *issuer); +extern jlib_decl IPropertyTree *getIssuerTlsServerConfigWithTrustedPeers(const char *issuer, const char *trusted_peers); + +extern jlib_decl IPropertyTree *createIssuerTlsClientConfig(const char *issuer, bool acceptSelfSigned, bool addCACert=true); extern jlib_decl void splitFullUrl(const char *url, bool &https, StringBuffer &user, StringBuffer &password, StringBuffer &host, StringBuffer &port, StringBuffer &fullpath); extern jlib_decl void splitUrlSchemeHostPort(const char *url, StringBuffer &user, StringBuffer &password, StringBuffer &schemeHostPort, StringBuffer &path); diff --git a/system/jlib/jsmartsock.cpp b/system/jlib/jsmartsock.cpp index 2c38088d2ea..7e074280ece 100644 --- a/system/jlib/jsmartsock.cpp +++ b/system/jlib/jsmartsock.cpp @@ -216,8 +216,9 @@ CSmartSocketFactory::CSmartSocketFactory(IPropertyTree &service, bool _retry, un throw createSmartSocketException(0, "CSmartSocket factory both name and port required for service configuration"); tlsService = service.getPropBool("@tls"); + issuer.set(service.queryProp("@issuer")); if (tlsService) - tlsConfig.setown(createTlsClientSecretInfo(service.queryProp("@issuer"), !service.getPropBool("@public"), service.getPropBool("@selfSigned"), service.getPropBool("@caCert"))); + tlsConfig.setown(createIssuerTlsClientConfig(issuer, service.getPropBool("@selfSigned"), service.getPropBool("@caCert"))); StringBuffer s; s.append(name).append(':').append(port); diff --git a/system/jlib/jsmartsock.ipp b/system/jlib/jsmartsock.ipp index 724f639d561..db59596ba03 100644 --- a/system/jlib/jsmartsock.ipp +++ b/system/jlib/jsmartsock.ipp @@ -69,6 +69,7 @@ protected: bool retry; bool tlsService = false; Owned tlsConfig; + StringAttr issuer; unsigned retryInterval; unsigned dnsInterval; @@ -104,6 +105,7 @@ public: virtual StringBuffer & getUrlStr(StringBuffer &str, bool useHostName); virtual bool isTlsService() const override { return tlsService; } virtual const IPropertyTree *queryTlsConfig() const { return tlsConfig; }; + const char *queryTlsIssuer() const { return issuer.str(); } }; diff --git a/system/mp/mpcomm.cpp b/system/mp/mpcomm.cpp index cf38e87afe9..5751b170b37 100644 --- a/system/mp/mpcomm.cpp +++ b/system/mp/mpcomm.cpp @@ -2132,7 +2132,7 @@ CMPConnectThread::CMPConnectThread(CMPServer *_parent, unsigned port, bool _list #if defined(_USE_OPENSSL) if (parent->useTLS) - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); #endif } diff --git a/system/security/securesocket/securesocket.cpp b/system/security/securesocket/securesocket.cpp index 37285487129..dfb8b0ca5ee 100644 --- a/system/security/securesocket/securesocket.cpp +++ b/system/security/securesocket/securesocket.cpp @@ -1985,9 +1985,9 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocket return new securesocket::CSecureSocketContext(ssf->queryTlsConfig(), ClientSocket); } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType sockettype) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *issuer, SecureSocketType sockettype) { - IPropertyTree *info = queryTlsSecretInfo(mtlsSecretName); + Owned info = getIssuerTlsServerConfig(issuer); //if the secret doesn't exist doesn't exist just go on without it. IF it is required the tls connection will fail. //This is primarily for client side... server side would probably use the explict ptree config or explict cert param at least for now. if (info) @@ -1996,16 +1996,16 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const cha return createSecureSocketContext(sockettype); } -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName) +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *issuer, bool requireMtlsFlag) { - if (!queryMtls()) + if (requireMtlsFlag && !queryMtls()) throw makeStringException(-100, "TLS secure communication requested but not configured"); - IPropertyTree *info = queryTlsSecretInfo(mtlsSecretName); - if (info) - return createSecureSocketContextEx2(info, ServerSocket); - else + Owned info = getIssuerTlsServerConfig(issuer); + if (!info) throw makeStringException(-101, "TLS secure communication requested but not configured (2)"); + + return createSecureSocketContextEx2(info, ServerSocket); } SECURESOCKET_API ICertificate *createCertificate() @@ -2172,7 +2172,7 @@ class CSingletonSecureSocketConnection: public CSingletonSocketConnection state = Snone; cancelling = false; secureContextClient.setown(createSecureSocketContextSecret("local", ClientSocket)); - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); #ifdef _CONTAINERIZED tlsLogLevel = getComponentConfigSP()->getPropInt("logging/@detail", SSLogMin); if (tlsLogLevel >= ExtraneousMsgThreshold) // or InfoMsgThreshold ? diff --git a/system/security/securesocket/securesocket.hpp b/system/security/securesocket/securesocket.hpp index cd3001b8460..2bf1758d92e 100644 --- a/system/security/securesocket/securesocket.hpp +++ b/system/security/securesocket/securesocket.hpp @@ -92,7 +92,7 @@ SECURESOCKET_API ISecureSocketContext* createSecureSocketContextEx(const char* c SECURESOCKET_API ISecureSocketContext* createSecureSocketContextEx2(const IPropertyTree* config, SecureSocketType); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSSF(ISmartSocketFactory* ssf); SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecret(const char *mtlsSecretName, SecureSocketType); -SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName); +SECURESOCKET_API ISecureSocketContext* createSecureSocketContextSecretSrv(const char *mtlsSecretName, bool requireMtlsConfig); SECURESOCKET_API ICertificate *createCertificate(); SECURESOCKET_API int signCertificate(const char* csr, const char* ca_certificate, const char* ca_privkey, const char* ca_passphrase, int days, StringBuffer& certificate); }; diff --git a/thorlcr/msort/tsorts1.cpp b/thorlcr/msort/tsorts1.cpp index 3e3ed4a3479..869d1271e8f 100644 --- a/thorlcr/msort/tsorts1.cpp +++ b/thorlcr/msort/tsorts1.cpp @@ -315,7 +315,7 @@ protected: friend class CSortMerge; #if defined(_USE_OPENSSL) if (slave.queryTLS()) { - secureContextServer.setown(createSecureSocketContextSecretSrv("local")); + secureContextServer.setown(createSecureSocketContextSecretSrv("local", true)); secureContextClients.setown(createSecureSocketContextSecret("local", ClientSocket)); } #endif