diff --git a/helm/examples/metrics/README.md b/helm/examples/metrics/README.md index e29fd25cc1c..387a756eea8 100644 --- a/helm/examples/metrics/README.md +++ b/helm/examples/metrics/README.md @@ -252,12 +252,41 @@ An example _yml_ file can be found in the repository at helm/examples/metrics/el Make a copy and modify as needed for your installation. ##### Configuration Settings -The ElasticSearch sink defines the following settings: +The ElasticSearch sink defines the following settings -* hostProtocol - The protocol used to connect to the ElasticSearch server. (default: https) -* hostName - The host name or IP address of the ElasticSearch server. (required) -* hostPort - The port number of the ElasticSearch server. (default: 9200) -* indexName - The name of the index to which metrics are reported. (required) +**Host** +The host settings define the ElasticSearch server to which metrics are reported. The settings are: + +* domain - The domain or IP address of the ElasticSearch server. (required) +* protocol - The protocol used to connect to the ElasticSearch server. (default: https) +* port - The port number of the ElasticSearch server. (default: 9200) +* certificateFilePath - Path to the file containing the certificate used to connect to the ElasticSearch server. (optional) + + +**Authentication** + +Optional child of the host configuration where authentication settings are defined. If missing, +no authentication is used. If defined, the settings are: + +* type - Required Authentication type used to connect to the ElasticSearch server. Value defines the +remaining settings. The allowed values are: + * basic - Basic authentication is used. +* credentialsSecret - The name of the secret containing the credentials used to authenticate to +the ElasticSearch server. (optional, valid for Kubernetes only) +* credentialsVaultId - The vault ID containing the credentials used to authenticate to the +ElasticSearch server. (optional, valid for Vault only) + +For **basic** authentication, the following settings are required, regardless if the credentials are stored +as a secret or in the environment.xml file. +* username - The username used to authenticate to the ElasticSearch server. +* password - The password used to authenticate to the ElasticSearch server. When stored in the +environment.xml file, it shall be encrypted using standard environment.xml encryption. + +**Index** + +The index settings define the index where metrics are indexed. The settings are: + +* name - The name of the index to which metrics are reported. (required) * countMetricSuffix - The suffix used to identify count metrics. (default: count) * gaugeMetricSuffix - The suffix used to identify gauge metrics. (default: gauge) * histogramMetricSuffix - The suffix used to identify histogram metrics. (default: histogram) @@ -269,17 +298,17 @@ To enable reporting of metrics to ElasticSearch, add the metric configuration se the environment configuration file (enviroment.xml). These settings must be added manually since there is no support in the config manager. -Add the following to the environment configuration file (note only the required -settings are shown): +Add the following to the environment.xml configuration file (note that some values may not be required): ```code xml - - + + + diff --git a/helm/examples/metrics/elasticsearch_metrics.yaml b/helm/examples/metrics/elasticsearch_metrics.yaml index 0b44d232b59..88c345978a5 100644 --- a/helm/examples/metrics/elasticsearch_metrics.yaml +++ b/helm/examples/metrics/elasticsearch_metrics.yaml @@ -1,17 +1,24 @@ # # Defines an elastic sink for reporting metrics to an ElasticSearch instance # Settings: -# type - sink type (must be elastic for ElasticSearch support) -# name - name for the sink instance -# settings.countMetricSuffix - suffix for count metrics (default: count) -# settings.gaugeMetricSuffix - suffix for gauge metrics (default: gauge) -# settings.histogramMetricSuffix - suffix for histogram metrics (default: histogram) -# settings.host - ElasticSearch host settings -# settings.host.protocol - protocol to use, http or https (default) -# settings.host.name - host name -# settings.host.port - port number (default 9200) -# settings.index - ElasticSearch index settings -# settings.index.name - index name +# type - sink type (must be elastic for ElasticSearch support) +# name - name for the sink instance +# settings.countMetricSuffix - suffix for count metrics (default: count) +# settings.gaugeMetricSuffix - suffix for gauge metrics (default: gauge) +# settings.histogramMetricSuffix - suffix for histogram metrics (default: histogram) +# settings.host - ElasticSearch host settings +# settings.host.protocol - protocol to use, http or https (default) +# settings.host.domain - host domain +# settings.host.certificateFilePath - path to certificate file (optional) +# settings.host.port - port number (default 9200) +# settings.host.authentication - authentication settings if authentication is enabled (optional) +# settings.host.authentication.type - authentication type (determines remaining settings) (only 'basic' is supported) +# settings.host.authentication.username - username for basic authentication (if not stored in a secret) +# settings.host.authentication.password - encrypted password for basic authentication (if not stored in a secret) +# settings.host.authentication.credentialsSecret - name of secret containing username and password for basic authentication +# settings.host.authentication.credentialsVaultId - optional vault id for secret containing username password for basic authentication +# settings.index - ElasticSearch index settings +# settings.index.name - index name # # If not overridden, the following suffixes are used by default: # countMetricSuffix: count @@ -29,8 +36,14 @@ global: histogramMetricSuffix: histogram host: protocol: https - name: hostname + domain: domain port: 9200 + certificateFilePath: "path/to/cert" + authentication: + type: basic + username: username + password: password + credentialsSecret: secretName + credentialsVaultId: vaultId index: name: hpccmetrics - diff --git a/system/metrics/sinks/elastic/elasticSink.cpp b/system/metrics/sinks/elastic/elasticSink.cpp index ceb0ff9fb87..989e45304b2 100644 --- a/system/metrics/sinks/elastic/elasticSink.cpp +++ b/system/metrics/sinks/elastic/elasticSink.cpp @@ -13,6 +13,8 @@ #include "elasticSink.hpp" #include "nlohmann/json.hpp" +#include "jsecrets.hpp" +#include "jencrypt.hpp" //including cpp-httplib single header file REST client // doesn't work with format-nonliteral as an error @@ -41,67 +43,136 @@ extern "C" MetricSink* getSinkInstance(const char *name, const IPropertyTree *pS ElasticMetricSink::ElasticMetricSink(const char *name, const IPropertyTree *pSettingsTree) : PeriodicMetricSink(name, "elastic", pSettingsTree) { + // Standard sink settings ignoreZeroMetrics = pSettingsTree->getPropBool("@ignoreZeroMetrics", true); - StringBuffer hostName; + // Get the host and index configuration + if (getHostConfig(pSettingsTree) && getIndexConfig(pSettingsTree)) + { + configurationValid = true; + PROGLOG("ElasticMetricSink: Loaded and configured"); + } +} + + +bool ElasticMetricSink::getHostConfig(const IPropertyTree *pSettingsTree) +{ + StringBuffer hostDomain; StringBuffer hostProtocol; StringBuffer hostPort; Owned pHostConfigTree = pSettingsTree->getPropTree("host"); if (pHostConfigTree) { - pHostConfigTree->getProp("@name", hostName); + pHostConfigTree->getProp("@domain", hostDomain); if (!pHostConfigTree->getProp("@protocol", hostProtocol)) - { hostProtocol.append("https"); - } if (!pHostConfigTree->getProp("@port", hostPort)) - { hostPort.append("9200"); - } } - if (!hostName.isEmpty() && !hostPort.isEmpty() && !hostProtocol.isEmpty()) + // Validate the host configuration minimal settings are present + if (hostDomain.isEmpty() || hostProtocol.isEmpty()) { - elasticHostUrl.append(hostProtocol).append("://").append(hostName).append(":").append(hostPort); + WARNLOG("ElasticMetricSink: Host configuration missing domain and/or protocol"); + return false; + } + + // build url for use with httplib Client + elasticHostUrl.append(hostProtocol).append("://").append(hostDomain); + if (!hostPort.isEmpty()) + elasticHostUrl.append(":").append(hostPort); + + // Read optional certificate file path + pHostConfigTree->getProp("@certificateFilePath", certificateFilePath); + + // Get authentication settings, if present + Owned pAuthConfigTree = pSettingsTree->getPropTree("authentication"); + if (!pAuthConfigTree) + return true; + + // Retrieve the authentication type and validate (only basic is supported) + if (!pAuthConfigTree->getProp("@type", authenticationType) || !streq(authenticationType, "basic")) + { + WARNLOG("ElasticMetricSink: Only basic authentication is supported"); + return false; + } + + StringBuffer credentialsSecretKey; + pAuthConfigTree->getProp("@credentialsSecret", credentialsSecretKey); // vault/secrets key + if (!credentialsSecretKey.isEmpty()) + { + StringBuffer credentialsVaultId; + pAuthConfigTree->getProp("@credentialsVaultId", credentialsVaultId);//optional HashiCorp vault ID + + PROGLOG("Retrieving ElasticSearch host authentication username/password from secrets tree '%s', from vault '%s'", + credentialsSecretKey.str(), !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : ""); + + Owned secretTree(getSecret("authn", credentialsSecretKey.str(), credentialsVaultId, nullptr)); + if (secretTree == nullptr) + { + WARNLOG("ElasticMetricSink: Unable to load secret tree '%s', from vault '%s'", credentialsSecretKey.str(), + !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : "n/a"); + return false; + } + + // authentication type defines the secret key name/value pairs to retrieve + if (streq(authenticationType, "basic")) + { + if (!getSecretKeyValue(username, secretTree, "username") || !getSecretKeyValue(password, secretTree, "password")) + { + WARNLOG("ElasticMetricSink: Missing username and/or password from secrets tree '%s', vault '%s'", + credentialsSecretKey.str(), !credentialsVaultId.isEmpty() ? credentialsVaultId.str() : "n/a"); + return false; + } + } } else { - WARNLOG("ElasticMetricSink: Host configuration missing or invalid"); + // if basic auth, username and password are stored directly in the configuration + if (streq(authenticationType, "basic")) + { + StringBuffer encryptedPassword; + if (!pAuthConfigTree->getProp("@username", username) || !pAuthConfigTree->getProp("@password", encryptedPassword)) + { + WARNLOG("ElasticMetricSink: Missing username and/or password from configuration"); + return false; + } + decrypt(password, encryptedPassword.str()); //MD5 encrypted in config + } } + return true; +} + +bool ElasticMetricSink::getIndexConfig(const IPropertyTree *pSettingsTree) +{ Owned pIndexConfigTree = pSettingsTree->getPropTree("index"); - if (pIndexConfigTree) + if (!pIndexConfigTree) { - pSettingsTree->getProp("@name", indexName); + WARNLOG("ElasticMetricSink: Index configuration missing"); + return false; } - if (indexName.isEmpty()) + if (!pIndexConfigTree->getProp("@name", indexName)) { - WARNLOG("ElasticMetricSink: Index configuration missing or invalid"); + WARNLOG("ElasticMetricSink: Index configuration missing name"); + return false; } - - // Both a host url and an index name are required - configurationValid = !elasticHostUrl.isEmpty() && !indexName.isEmpty(); - // Initialize standard suffixes - if (!pSettingsTree->getProp("@countMetricSuffix", countMetricSuffix)) - { + if (!pIndexConfigTree->getProp("@countMetricSuffix", countMetricSuffix)) countMetricSuffix.append("count"); - } if (!pSettingsTree->getProp("@gaugeMetricSuffix", gaugeMetricSuffix)) - { gaugeMetricSuffix.append("gauge"); - } if (!pSettingsTree->getProp("@histogramMetricSuffix", histogramMetricSuffix)) - { histogramMetricSuffix.append("histogram"); - } + + return true; } diff --git a/system/metrics/sinks/elastic/elasticSink.hpp b/system/metrics/sinks/elastic/elasticSink.hpp index c435ed04abb..95b1f9b275b 100644 --- a/system/metrics/sinks/elastic/elasticSink.hpp +++ b/system/metrics/sinks/elastic/elasticSink.hpp @@ -34,11 +34,17 @@ class ELASTICSINK_API ElasticMetricSink : public hpccMetrics::PeriodicMetricSink virtual void prepareToStartCollecting() override; virtual void collectingHasStopped() override; virtual void doCollection() override; + bool getHostConfig(const IPropertyTree *pSettingsTree); + bool getIndexConfig(const IPropertyTree *pSettingsTree); protected: StringBuffer indexName; bool ignoreZeroMetrics = false; StringBuffer elasticHostUrl; + StringBuffer certificateFilePath; + StringBuffer authenticationType; + StringBuffer username; + StringBuffer password; StringBuffer countMetricSuffix; StringBuffer gaugeMetricSuffix; StringBuffer histogramMetricSuffix;