Merge pull request hpcc-systems#19193 from dcamper/hpcc-32108-regex-c…

…ache-stats HPCC-32108 Gather and report regex cache statistics Reviewed-By: Shamser Ahmed <[email protected]> Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
kenrowland · Oct 16, 2024 · 474e817 · 474e817
2 parents d75da84 + 0f6e244
commit 474e817
Show file tree

Hide file tree

Showing 26 changed files with 197 additions and 48 deletions.
diff --git a/common/thorhelper/roxiehelper.cpp b/common/thorhelper/roxiehelper.cpp
@@ -2875,6 +2875,9 @@ class NullSectionTimer : public CSimpleInterfaceOf<ISectionTimer>
 {
     virtual unsigned __int64 getStartCycles() { return 0; }
     virtual void noteSectionTime(unsigned __int64 startCycles) {}
+    virtual void addStatistic(__int64 kind, unsigned __int64 value) {}
+    virtual void setStatistic(__int64 kind, unsigned __int64 value) {}
+    virtual void mergeStatistic(__int64 kind, unsigned __int64 value) {}
 };
 
 static NullSectionTimer nullSectionTimer;

diff --git a/common/thorhelper/thorstats.cpp b/common/thorhelper/thorstats.cpp
@@ -20,34 +20,63 @@
 #include "thorstats.hpp"
 #include "jdebug.hpp"
 
+// Available sets of statistics for a nested section
+const StatisticsMapping defaultNestedSectionStatistics({StCycleLocalExecuteCycles, StTimeLocalExecute, StNumStarts, StNumStops});
+const StatisticsMapping genericCacheNestedSectionStatistics({StNumCacheAdds, StNumCacheHits, StNumPeakCacheObjects}, defaultNestedSectionStatistics);
 
-//Cycles are accumulated locally, time is updated once it is serialized or persisted
-const StatisticsMapping nestedSectionStatistics({StCycleLocalExecuteCycles, StTimeLocalExecute, StNumStarts, StNumStops});
-
-ThorSectionTimer::ThorSectionTimer(const char * _name, CRuntimeStatistic & _starts, CRuntimeStatistic & _stops, CRuntimeStatistic & _elapsed)
-: starts(_starts), stops(_stops), elapsed(_elapsed), name(_name)
+ThorSectionTimer::ThorSectionTimer(const char * _name, CRuntimeStatisticCollection & _stats)
+: name(_name), stats(_stats)
 {
 }
 
-ThorSectionTimer * ThorSectionTimer::createTimer(CRuntimeStatisticCollection & stats, const char * name)
+ThorSectionTimer * ThorSectionTimer::createTimer(CRuntimeStatisticCollection & stats, const char * name, const StatisticsMapping & nestedSectionStatistics)
 {
     StatsScopeId scope(SSTfunction, name);
     CRuntimeStatisticCollection & nested = stats.registerNested(scope, nestedSectionStatistics);
-    CRuntimeStatistic & starts = nested.queryStatistic(StNumStarts);
-    CRuntimeStatistic & stops = nested.queryStatistic(StNumStops);
-    CRuntimeStatistic & elapsed = nested.queryStatistic(StCycleLocalExecuteCycles);
-    return new ThorSectionTimer(name, starts, stops, elapsed);
+    return new ThorSectionTimer(name, nested);
+}
+
+ThorSectionTimer * ThorSectionTimer::createTimer(CRuntimeStatisticCollection & stats, const char * name, ThorStatOption statOption)
+{
+    assertex(statOption < ThorStatMax);
+    switch (statOption)
+    {
+        case ThorStatGenericCache:
+            return createTimer(stats, name, genericCacheNestedSectionStatistics);
+        default:
+            return createTimer(stats, name, defaultNestedSectionStatistics);
+    }
+}
+
+ThorSectionTimer * ThorSectionTimer::createTimer(CRuntimeStatisticCollection & stats, const char * name)
+{
+    return createTimer(stats, name, defaultNestedSectionStatistics);
 }
 
 unsigned __int64 ThorSectionTimer::getStartCycles()
 {
-    starts.addAtomic(1);
+    stats.queryStatistic(StNumStarts).addAtomic(1);
     return get_cycles_now();
 }
 
 void ThorSectionTimer::noteSectionTime(unsigned __int64 startCycles)
 {
     cycle_t delay = get_cycles_now() - startCycles;
-    elapsed.addAtomic(delay);
-    stops.addAtomic(1);
+    stats.addStatisticAtomic(StCycleLocalExecuteCycles, delay);
+    stats.addStatisticAtomic(StNumStops, 1);
+}
+
+void ThorSectionTimer::addStatistic(__int64 kind, unsigned __int64 value)
+{
+    stats.addStatisticAtomic(static_cast<StatisticKind>(kind), value);
+}
+
+void ThorSectionTimer::setStatistic(__int64 kind, unsigned __int64 value)
+{
+    stats.setStatistic(static_cast<StatisticKind>(kind), value);
+}
+
+void ThorSectionTimer::mergeStatistic(__int64 kind, unsigned __int64 value)
+{
+    stats.mergeStatistic(static_cast<StatisticKind>(kind), value);
 }
diff --git a/common/thorhelper/thorstats.hpp b/common/thorhelper/thorstats.hpp
@@ -24,9 +24,22 @@
 #include "eclhelper.hpp"
 #include "jstats.h"
 
+// If you add values to this enum, update ThorSectionTimer::createTimer so that it can handle the new value
+enum ThorStatOption
+{
+    ThorStatDefault,
+    ThorStatGenericCache,
+    ThorStatMax // Marker, always last
+};
+
 class THORHELPER_API ThorSectionTimer : public CSimpleInterfaceOf<ISectionTimer>
 {
 public:
+    // Creates an instance of the timer that tracks the statistics cited in nestedSectionStatistics
+    static ThorSectionTimer * createTimer(CRuntimeStatisticCollection & _stats, const char * _name, const StatisticsMapping & nestedSectionStatistics);
+    // Creates an instance of the timer that tracks the statistics referenced through the ThorStatOption enum
+    static ThorSectionTimer * createTimer(CRuntimeStatisticCollection & _stats, const char * _name, ThorStatOption statOption);
+    // Default instantiation of a timer; uses defaultNestedSectionStatistics
     static ThorSectionTimer * createTimer(CRuntimeStatisticCollection & _stats, const char * _name);
 
     bool matches(const char * _name) const
@@ -36,15 +49,16 @@ class THORHELPER_API ThorSectionTimer : public CSimpleInterfaceOf<ISectionTimer>
 
     virtual unsigned __int64 getStartCycles();
     virtual void noteSectionTime(unsigned __int64 startCycles);
+    virtual void addStatistic(__int64 kind, unsigned __int64 value) override;
+    virtual void setStatistic(__int64 kind, unsigned __int64 value) override;
+    virtual void mergeStatistic(__int64 kind, unsigned __int64 value) override;
 
 protected:
-    ThorSectionTimer(const char * _name, CRuntimeStatistic & _starts, CRuntimeStatistic & _stops, CRuntimeStatistic & _elapsed);
+    ThorSectionTimer(const char * _name, CRuntimeStatisticCollection & _stats);
 
 private:
-    CRuntimeStatistic & starts;
-    CRuntimeStatistic & stops;
-    CRuntimeStatistic & elapsed;
     StringAttr name;
+    CRuntimeStatisticCollection & stats;
 };
 
 #endif
diff --git a/ecl/eclagent/eclagent.ipp b/ecl/eclagent/eclagent.ipp
@@ -547,6 +547,11 @@ public:
     {
         return queryNullSectionTimer();
     }
+
+    virtual ISectionTimer * registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption)
+    {
+        return queryNullSectionTimer();
+    }
 // IEngineContext
     virtual DALI_UID getGlobalUniqueIds(unsigned num, SocketEndpoint *_foreignNode)
     {

diff --git a/ecl/hql/hqlfold.cpp b/ecl/hql/hqlfold.cpp
@@ -1664,6 +1664,7 @@ class DummyContext: implements ICodeContext
     virtual IEngineRowAllocator * getRowAllocatorEx(IOutputMetaData * meta, unsigned activityId, unsigned flags) const { throwUnexpected(); }
     virtual void addWuExceptionEx(const char*, unsigned int, unsigned int, unsigned int, const char*) override { throwUnexpected(); }
     virtual unsigned getElapsedMs() const  override { throwUnexpected(); }
+    virtual ISectionTimer * registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption) { throwUnexpected(); }
 
 };
 

diff --git a/ecl/hqlcpp/hqlcatom.cpp b/ecl/hqlcpp/hqlcatom.cpp
@@ -552,6 +552,7 @@ IIdAtom * regexUStrMatchSetId;
 IIdAtom * regexU8StrMatchSetId;
 IIdAtom * regexReplaceXId;
 IIdAtom * registerTimerId;
+IIdAtom * registerStatsTimerId;
 IIdAtom * releaseRowId;
 IIdAtom * releaseRowsetId;
 IIdAtom * reportFieldOverflowId;
@@ -1243,6 +1244,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM-1)
     MAKEID(regexU8StrMatchSet);
     MAKEID(regexReplaceX);
     MAKEID(registerTimer);
+    MAKEID(registerStatsTimer);
     MAKEID(releaseRow);
     MAKEID(releaseRowset);
     MAKEID(reportFieldOverflow);

diff --git a/ecl/hqlcpp/hqlcatom.hpp b/ecl/hqlcpp/hqlcatom.hpp
@@ -550,6 +550,7 @@ extern IIdAtom * regexUStrMatchSetId;
 extern IIdAtom * regexU8StrMatchSetId;
 extern IIdAtom * regexReplaceXId;
 extern IIdAtom * registerTimerId;
+extern IIdAtom * registerStatsTimerId;
 extern IIdAtom * releaseRowId;
 extern IIdAtom * releaseRowsetId;
 extern IIdAtom * reportFieldOverflowId;

diff --git a/ecl/hqlcpp/hqlcpp.cpp b/ecl/hqlcpp/hqlcpp.cpp
@@ -2248,7 +2248,9 @@ void HqlCppTranslator::callProcedure(BuildCtx & ctx, IIdAtom * name, HqlExprArra
         const char * timerName = str(external->queryId());
         if (getStringValue(nameTemp, queryAttributeChild(external, timerAtom, 0)).length())
             timerName = nameTemp;
-        buildHelperTimer(ctx, boundTimer, timerName);
+        // Grab optional ThorStatOption enum (as an int)
+        int statOption = getIntValue(queryAttributeChild(external, timerAtom, 1), 0);
+        buildHelperTimer(ctx, boundTimer, timerName, statOption);
         // We need to patch the args to include the timer; helper timers will always be the first argument
         // passed to the helper function, which means that it is either the first argument in the
         // array (for a bare function call) or the second argument (for a method call)
@@ -6127,7 +6129,9 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
         const char * name = str(external->queryId());
         if (getStringValue(nameTemp, queryAttributeChild(external, timerAtom, 0)).length())
             name = nameTemp;
-        buildHelperTimer(ctx, boundTimer, name);
+        // Grab optional ThorStatOption enum (as an int)
+        int statOption = getIntValue(queryAttributeChild(external, timerAtom, 1), 0);
+        buildHelperTimer(ctx, boundTimer, name, statOption);
         args.append(*LINK(boundTimer.expr));
     }
     if (external->hasAttribute(userMatchFunctionAtom))

diff --git a/ecl/hqlcpp/hqlcpp.ipp b/ecl/hqlcpp/hqlcpp.ipp
@@ -1676,8 +1676,8 @@ public:
     IHqlExpression * getFirstCharacter(IHqlExpression * source);
     bool hasAddress(BuildCtx & ctx, IHqlExpression * expr);
 
-    void buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name);
-    void buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name);
+    void buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name, int statsOption);
+    void buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name, int statsOption);
     void buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, CHqlBoundExpr & boundStart, const char * name);
     void buildStopTimer(BuildCtx & ctx, const CHqlBoundExpr & boundTimer, const CHqlBoundExpr & boundStart);
 

diff --git a/ecl/hqlcpp/hqlcppsys.ecl b/ecl/hqlcpp/hqlcppsys.ecl
@@ -553,7 +553,7 @@ const char * cppSystemText[]  = {
     "   unicode wregexReplaceX(unsigned8 compiled, const unicode text, const unicode replace)   :   eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlWRegExprReplace';",
     "   unicode wregexGetFindStr(unsigned8 compiled, unsigned4 idx):    eclrtl,pure,include='eclrtl.hpp',library='eclrtl',entrypoint='rtlWRegExprGetFindStr';",
 
-    "   regexNewSetStrPattern(const varstring _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileStringRegex');"
+    "   regexNewSetStrPattern(const varstring _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileStringRegex', 1);"
     "   regexNewStrFind(boolean _compiled, const string _search, boolean _cloneSearch) : omethod,entrypoint='findTimed',timer('REGEXFIND');"
     "   boolean regexNewStrFound() : method,pure,entrypoint='found';"
     "   string regexNewStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
@@ -562,7 +562,7 @@ const char * cppSystemText[]  = {
     "   regexNewStrReplaceFixed(noconst string _tgt, const string _search, const string _replace) : method,pure,entrypoint='replaceFixedTimed',timer('REGEXREPLACE');"
     "   set of string regexMatchSet(const string _search) : method,pure,entrypoint='getMatchSetTimed',timer('REGEXFINDSET');"
 
-    "   regexNewSetUStrPattern(const varunicode _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUnicodeRegex');"
+    "   regexNewSetUStrPattern(const varunicode _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUnicodeRegex', 1);"
     "   regexNewUStrFind(boolean _compiled, const unicode _search) : omethod,entrypoint='findTimed',timer('REGEXFIND');"
     "   boolean regexNewUStrFound() : method,pure,entrypoint='found';"
     "   unicode regexNewUStrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
@@ -571,7 +571,7 @@ const char * cppSystemText[]  = {
     "   regexNewUStrReplaceFixed(noconst unicode _tgt, const unicode _search, const unicode _replace) : method,pure,entrypoint='replaceFixedTimed',timer('REGEXREPLACE');"
     "   set of unicode regexUStrMatchSet(const unicode _search) : method,pure,entrypoint='getMatchSetTimed',timer('REGEXFINDSET');"
 
-    "   regexNewSetU8StrPattern(const utf8 _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUTF8Regex');"
+    "   regexNewSetU8StrPattern(const utf8 _pattern, boolean isCaseSensitive) : omethod,entrypoint='setPatternTimed',timer('CompileUTF8Regex', 1);"
     "   regexNewU8StrFind(boolean _compiled, const utf8 _search, boolean _cloneSearch) : omethod,entrypoint='findTimed',timer('REGEXFIND');"
     "   boolean regexNewU8StrFound() : method,pure,entrypoint='found';"
     "   utf8 regexNewU8StrFoundX(unsigned4 idx) : method,pure,entrypoint='getMatchX';"
@@ -915,6 +915,7 @@ const char * cppSystemText[]  = {
     "   varstring registerTimer(unsigned4 id, const varstring name) : ctxmethod;",
     "   unsigned8 getStartCycles() : method;",
     "   noteSectionTime(unsigned8 started) : method;",
+    "   varstring registerStatsTimer(unsigned4 id, const varstring name, unsigned4 statsOption) : ctxmethod;",
     "   END;",
     NULL };
 

diff --git a/ecl/hqlcpp/hqlhtcpp.cpp b/ecl/hqlcpp/hqlhtcpp.cpp
@@ -18814,7 +18814,7 @@ void HqlCppTranslator::doBuildExprRegexFindSet(BuildCtx & ctx, IHqlExpression *
 
 //---------------------------------------------------------------------------
 
-void HqlCppTranslator::buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name)
+void HqlCppTranslator::buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name, int statsOption)
 {
     BuildCtx * initCtx = &ctx;
     BuildCtx * declareCtx = &ctx;
@@ -18828,7 +18828,16 @@ void HqlCppTranslator::buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer
     HqlExprArray registerArgs;
     registerArgs.append(*getSizetConstant(activityId));
     registerArgs.append(*createConstant(name));
-    OwnedHqlExpr call = bindFunctionCall(registerTimerId, registerArgs);
+    OwnedHqlExpr call;
+    if (statsOption == 0) // enum ThorStatOption.ThorStatDefault
+    {
+        call.setown(bindFunctionCall(registerTimerId, registerArgs));
+    }
+    else
+    {
+        registerArgs.append(*createConstant(statsOption));
+        call.setown(bindFunctionCall(registerStatsTimerId, registerArgs));
+    }
 
     if (!declareCtx->getMatchExpr(call, boundTimer))
     {
@@ -18840,14 +18849,14 @@ void HqlCppTranslator::buildTimerBase(BuildCtx & ctx, CHqlBoundExpr & boundTimer
     }
 }
 
-void HqlCppTranslator::buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name)
+void HqlCppTranslator::buildHelperTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, const char * name, int statsOption)
 {
-    buildTimerBase(ctx, boundTimer, name);
+    buildTimerBase(ctx, boundTimer, name, statsOption);
 }
 
 void HqlCppTranslator::buildStartTimer(BuildCtx & ctx, CHqlBoundExpr & boundTimer, CHqlBoundExpr & boundStart, const char * name)
 {
-    buildTimerBase(ctx, boundTimer, name);
+    buildTimerBase(ctx, boundTimer, name, 0);
 
     HqlExprArray nowArgs;
     nowArgs.append(*boundTimer.getTranslatedExpr());

diff --git a/roxie/ccd/ccdactivities.cpp b/roxie/ccd/ccdactivities.cpp
@@ -564,6 +564,11 @@ class CRoxieAgentActivity : implements CInterfaceOf<IRoxieAgentActivity>, implem
         return queryNullSectionTimer();
     }
 
+    virtual ISectionTimer * registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption)
+    {
+        return queryNullSectionTimer();
+    }
+
     // Not yet thought about these....
 
     virtual char *getWuid() { throwUnexpected(); } // caller frees return string.

diff --git a/roxie/ccd/ccdcontext.cpp b/roxie/ccd/ccdcontext.cpp
@@ -2136,12 +2136,16 @@ class CRoxieContextBase : implements IRoxieAgentContext, implements ICodeContext
         return rtlVCodepageToVUnicodeX(x.str(), "utf-8");
     }
     virtual ISectionTimer * registerTimer(unsigned activityId, const char * name)
+    {
+        return registerStatsTimer(activityId, name, 0);
+    }
+    virtual ISectionTimer * registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption)
     {
         CriticalBlock b(timerCrit);
         ISectionTimer *timer = functionTimers.getValue(name);
         if (!timer)
         {
-            timer = ThorSectionTimer::createTimer(globalStats, name);
+            timer = ThorSectionTimer::createTimer(globalStats, name, static_cast<ThorStatOption>(statsOption));
             functionTimers.setValue(name, timer);
             timer->Release(); // Value returned is not linked
         }

diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp
@@ -1109,6 +1109,13 @@ class CRoxieServerActivity : implements CInterfaceOf<IRoxieServerActivity>, impl
             else
                 return ctx->registerTimer(activityId, name);
         }
+        virtual ISectionTimer *registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption)
+        {
+            if (activityId == activity.queryId())
+                return activity.registerStatsTimer(activityId, name, statsOption);
+            else
+                return ctx->registerStatsTimer(activityId, name, statsOption);
+        }
     private:
         CRoxieServerActivity &activity;
     } interceptedCtx;
@@ -1298,12 +1305,16 @@ class CRoxieServerActivity : implements CInterfaceOf<IRoxieServerActivity>, impl
         return stats.toStr(ret);
     }
     virtual ISectionTimer *registerTimer(unsigned _activityId, const char * name)
+    {
+        return registerStatsTimer(_activityId, name, 0);
+    }
+    virtual ISectionTimer *registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption)
     {
         CriticalBlock b(statscrit); // reuse statscrit to protect functionTimers - it will not be held concurrently
         ISectionTimer *timer = functionTimers.getValue(name);
         if (!timer)
         {
-            timer = ThorSectionTimer::createTimer(stats, name);
+            timer = ThorSectionTimer::createTimer(stats, name, static_cast<ThorStatOption>(statsOption));
             functionTimers.setValue(name, timer);
             timer->Release(); // Value returned is not linked
         }

diff --git a/roxie/ccd/ccdserver.hpp b/roxie/ccd/ccdserver.hpp
@@ -204,6 +204,7 @@ interface IRoxieServerActivity : extends IActivityBase
     virtual void gatherStatistics(IStatisticGatherer * statsBuilder) const = 0;
     virtual void noteStatistic(StatisticKind kind, unsigned __int64 value) const = 0;
     virtual void noteLibrary(IQueryFactory *library) = 0;
+    virtual ISectionTimer * registerStatsTimer(unsigned activityId, const char * name, unsigned int statsOption) = 0;
 };
 
 interface IRoxieServerActivityFactory : extends IActivityFactory