HPCC-32465 Add ESP support for trace level

- Define an ESP process configuration node that supports specification of global
  TraceFlags values for each ESP.
- Reserve traceLevel to request a specific trace level (0: none, 1: standard,
  2: detailed, 3: max). This replaces acting on the last observed occurrence
  of either traceNone, traceStandard, traceDetailed, and traceMax.
- Override default flag settings when not configured and a debug build.

Signed-off-by: Tim Klemm <[email protected]>

esp/platform/application_config.cpp

@@ -27,6 +27,7 @@
 #include "espcfg.ipp"
 #include "esplog.hpp"
 #include "espcontext.hpp"
+#include "esptrace.h"
 
 enum class LdapType { LegacyAD, AzureAD };
 
@@ -436,6 +437,18 @@ void setLDAPSecurityInWSAccess(IPropertyTree *legacyEsp, IPropertyTree *legacyLd
     }
 }
 
+// Copy trace flags from appEsp to legacyEsp. The source is expected to be an application's `esp`
+// configuration object. The destination is expected to be the `EspProcess` element.
+void copyTraceFlags(IPropertyTree *legacyEsp, IPropertyTree *appEsp)
+{
+    IPropertyTree *traceFlags = appEsp->queryPropTree(propTraceFlags);
+    if (traceFlags)
+    {
+        IPropertyTree *legacyTraceFlags = legacyEsp->addPropTree(propTraceFlags);
+        copyAttributes(legacyTraceFlags, traceFlags);
+    }
+}
+
 IPropertyTree *buildApplicationLegacyConfig(const char *application, const char* argv[])
 {
     Owned<IPropertyTree> appEspConfig = loadApplicationConfig(application, argv);
@@ -468,5 +481,7 @@ IPropertyTree *buildApplicationLegacyConfig(const char *application, const char*
     IPropertyTree *legacyDirectories = legacy->queryPropTree("Software/Directories");
     IPropertyTree *appDirectories = appEspConfig->queryPropTree("directories");
     copyDirectories(legacyDirectories, appDirectories);
+
+    copyTraceFlags(legacyEsp, appEspConfig);
     return legacy.getClear();
 }

esp/platform/espp.cpp

@@ -49,6 +49,10 @@
 
 #include "jmetrics.hpp"
 #include "workunit.hpp"
+#include "esptrace.h"
+#include "tokenserialization.hpp"
+
+static TokenDeserializer deserializer;
 
 using namespace hpccMetrics;
 
@@ -354,6 +358,91 @@ static void usage()
 
 IPropertyTree *buildApplicationLegacyConfig(const char *application, const char* argv[]);
 
+// Modified version of jlib's loadTraceFlags. The modification adds special handling for traceLevel.
+//
+// Using loadTraceFlags, `traceStandard: true`, `traceStandard: false`, and `traceStandard: maybe`
+// are non-intuitively equivalent. The property name is the only thing that matters, with the
+// value being ignored.
+//
+// Also using loadTraceFlags, all defined trace levels may be included in one configuration. The
+// final, accepted, level is determined by the order of `optNames` list items. An order optimized
+// for elevating the level cannot be used in reverse. While a configuration with multiple properties
+// should be discouraged, using `helm install --set` to change the level will add a new property
+// rather than changing the existing one.
+//
+// With loadEspTraceFlags, `traceLevel` is the only property that sets the trace level. The value
+// is both relevant and intuitive, and can be overridden with `helm install --set`.
+static TraceFlags loadEspTraceFlags(const IPropertyTree *ptree, const std::initializer_list<TraceOption> &optNames, TraceFlags dft)
+{
+    for (const TraceOption& option: optNames)
+    {
+        VStringBuffer attrName("@%s", option.name);
+        const char* value = nullptr;
+        if (!(value = ptree->queryProp(attrName)))
+        {
+            attrName.setCharAt(0, '_');
+            if (!(value = ptree->queryProp(attrName)))
+                continue;
+        }
+        if (streq(value, "default")) // allow a configuration to explicitly request a default value
+            continue;
+        if (streq(option.name, propTraceLevel)) // non-Boolean traceLevel
+        {
+            unsigned level = unsigned(dft & TraceFlags::LevelMask);
+            dft &= ~TraceFlags::LevelMask;
+            if (streq(value, "traceStandard"))
+                dft |= traceStandard;
+            else if (streq(value, "traceDetailed"))
+                dft |= traceDetailed;
+            else if (streq(value, "traceMax"))
+                dft |= traceMax;
+            else if (deserializer(value, level) == Deserialization_SUCCESS)
+                dft |= TraceFlags(level) & TraceFlags::LevelMask;
+            else
+                dft |= TraceFlags(level);
+        }
+        else if (option.value <= TraceFlags::LevelMask) // block individual trace level names
+            continue;
+        else // Boolean trace options
+        {
+            bool flag = false;
+            deserializer(value, flag);
+            if (flag)
+                dft |= option.value;
+            else
+                dft &= ~option.value;
+        }
+    }
+    return dft;
+}
+
+//
+// Initialize trace settings
+void initializeTraceFlags(CEspConfig* config)
+{
+    IPropertyTree* pEspTree = config->queryConfigPTree();
+    Owned<IPropertyTree> pTraceTree = pEspTree->getPropTree(propTraceFlags);
+    if (!pTraceTree)
+    {
+        pTraceTree.setown(getComponentConfigSP()->getPropTree(propTraceFlags));
+    }
+#ifdef _DEBUG
+    if (!pTraceTree)
+    {
+        static const char * defaultTraceYaml = R"!!(
+traceLevel: traceMax
+)!!";
+        pTraceTree.setown(createPTreeFromYAMLString(defaultTraceYaml));
+    }
+#endif
+
+    if (pTraceTree)
+    {
+        TraceFlags defaults = loadEspTraceFlags(pTraceTree, mapTraceOptions(pEspTree), queryDefaultTraceFlags());
+        updateTraceFlags(defaults, true);
+    }
+}
+
 //
 // Initialize metrics
 void initializeMetrics(CEspConfig* config)
@@ -577,6 +666,7 @@ int init_main(int argc, const char* argv[])
             config->bindServer(*server.get(), *server.get());
             config->checkESPCache(*server.get());
 
+            initializeTraceFlags(config);
             initializeMetrics(config);        
             initializeStorageGroups(daliClientActive());
         }

esp/platform/esptrace.h

@@ -0,0 +1,73 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2024 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#pragma once
+
+#include "jtrace.hpp"
+#include <initializer_list>
+
+// Refer to helm/examples/esp/README.md for more information on the trace options.
+
+constexpr const char* propTraceFlags = "traceFlags";
+constexpr const char* propTraceLevel = "traceLevel";
+
+// ESP process names for use with options mapping
+constexpr const char* eclwatchApplication = "eclwatch";
+constexpr const char* eclservicesApplication = "eclservices";
+constexpr const char* eclqueriesApplication = "eclqueires";
+constexpr const char* esdlsandboxApplication = "esdl-sandbox";
+constexpr const char* esdlApplication = "esdl";
+constexpr const char* sql2eclApplication = "sql2ecl";
+constexpr const char* dfsApplication = "dfs";
+constexpr const char* ldapenvironmentApplication = "ldapenvironment";
+constexpr const char* loggingserviceApplication = "loggingservice";
+
+// Trace options for ESP
+constexpr TraceFlags traceLevel = TraceFlags::LevelMask;
+
+// Trace option list fragment for jtrace-defined options used by ESPs
+#define PLATFORM_OPTIONS_FRAGMENT
+
+// Trace option list fragment for options used by most ESPs
+#define ESP_OPTIONS_FRAGMENT \
+    PLATFORM_OPTIONS_FRAGMENT \
+    TRACEOPT(traceLevel),
+
+// Trace option initializer list for ESPs that do not define their own options.
+constexpr std::initializer_list<TraceOption> espTraceOptions
+{
+    ESP_OPTIONS_FRAGMENT
+};
+
+/**
+ * @brief Returns the trace options appropriate for the ESP process being initialized.
+ *
+ * Most ESPs will simply return espTraceOptions. Any ESP that defines options not applicable to
+ * other ESPs would return a different list. The determination of which list to return is
+ * expected to be based on the configuration's `application` property.
+ *
+ * If options for all ESPs are defined with no value collisions, there may be no need to define
+ * separate option lists for individual ESPs. However, if value collisions cannot be avoided,
+ * separate lists will be needed.
+ *
+ * Consider ESDL Script, and the applications that use it. The potential for a significant number
+ * of options is high, increasing the likelihood of collisions with applications that don't use it.
+ */
+inline const std::initializer_list<TraceOption>& mapTraceOptions(const IPTree* config)
+{
+    return espTraceOptions;
+}

helm/examples/esp/README.md

@@ -0,0 +1,62 @@
+# ESP Trace Flags
+
+Each ESP process may specify its own set of flags for controlling trace behavior. The specific flags may be shared with other platform components, shared amongst ESPs, or unique to an ESP.
+
+## Accepted Flags
+
+Detailed description of flags used by any ESP.
+
+### Shared Platform Flags
+
+Flags defined in `system/jlib/jtrace.hpp` and used by multiple platform processes.
+
+Flags will be added to this list as tracing logic is updated in ESP code. For example, the shared platform flag `traceHttp` is expected to be used, as are a number of ESP-specific options.
+
+### Shared ESP Flags
+
+Flags defined in `esp/platform/esptrace.h` and applicable to most, if not all, ESP configurations.
+
+#### traceLevel
+
+Set the defailt trace level in the process. Accepted values are:
+- 1, traceStandard: most important output
+- 2, traceDetailed: average verbosity output
+- 3, traceMax: highest verbosity output
+- default: use existing level
+  - Release builds default to traceStandard
+  - Debug builds default to traceMax
+- 0, traceNone, *all other values*: no trace output
+
+## Process Configuration
+
+### Containerized
+
+#### esp.yaml
+
+Each ESP application's configuration object may embed one instance of a `traceFlags` object. Within this object, at most one property per [available flag]]#availableflags] is expected. Properties not described here are ignored.
+
+For example, the `eclwatch` process might be configured to use detailed reporting like this:
+
+```yml
+esp:
+- name: eclwatch
+  traceFlags:
+    traceLevel: 2
+```
+
+## Cluster Overrides
+
+A values file may be used with the `helm install` command to override the settings of all ESPs. The `--set` option may be used to target the settings of a specific ESP in the configured array.
+
+### Bare-Metal
+
+No support for defining trace flags is provided by the `configmgr` application. Within a stand-alone `esp.xml` file, however, a `traceFlags` child of the `EspProcess` element may be defined.
+
+The previous YAML example may be reproduced in XML with the following:
+
+```xml
+<EspProcess ...>
+  <traceFlags traceLevel="2" />
+  ...
+<EspProcess>
+```