Merge pull request #18920 from jakesmith/HPCC-32250-peak-lookahead

HPCC-32250 Ensure lookahead spill stats are published Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Jul 26, 2024 · d2a4f1c · d2a4f1c
2 parents b32d940 + 1ea7b7d
commit d2a4f1c
Show file tree

Hide file tree

Showing 7 changed files with 10 additions and 15 deletions.
diff --git a/thorlcr/activities/loop/thloop.cpp b/thorlcr/activities/loop/thloop.cpp
@@ -334,7 +334,7 @@ class CLocalResultActivityMasterBase : public CMasterActivity
     Owned<IThorRowInterfaces> inputRowIf;
 
 public:
-    CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info, spillingActivityStatistics)
+    CLocalResultActivityMasterBase(CMasterGraphElement *info) : CMasterActivity(info)
     {
     }
     virtual void init() override

diff --git a/thorlcr/activities/loop/thloopslave.cpp b/thorlcr/activities/loop/thloopslave.cpp
@@ -643,7 +643,7 @@ class CLocalResultSpillActivity : public CSlaveActivity
     }
 
 public:
-    CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics)
+    CLocalResultSpillActivity(CGraphElementBase *_container) : CSlaveActivity(_container)
     {
         helper = (IHThorLocalResultSpillArg *)queryHelper();
         appendOutputLinked(this);
@@ -708,7 +708,7 @@ class CLocalResultSpillActivity : public CSlaveActivity
 class CLocalResultWriteActivityBase : public ProcessSlaveActivity
 {
 public:
-    CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container, spillingActivityStatistics)
+    CLocalResultWriteActivityBase(CGraphElementBase *_container) : ProcessSlaveActivity(_container)
     {
     }
     virtual IThorResult *createResult() = 0;

diff --git a/thorlcr/activities/nsplitter/thnsplitterslave.cpp b/thorlcr/activities/nsplitter/thnsplitterslave.cpp
@@ -152,7 +152,7 @@ class NSplitterSlaveActivity : public CSlaveActivity, implements ISharedSmartBuf
         }
     }
 public:
-    NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container, spillingActivityStatistics), writer(*this)
+    NSplitterSlaveActivity(CGraphElementBase *_container) : CSlaveActivity(_container), writer(*this)
     {
         numOutputs = container.getOutputs();
         connectedOutputSet.setown(createBitSet());

diff --git a/thorlcr/master/thactivitymaster.cpp b/thorlcr/master/thactivitymaster.cpp
@@ -207,10 +207,8 @@ class CGenericMasterGraphElement : public CMasterGraphElement
             case TAKdistributed:
             case TAKtrace:
             case TAKemptyaction:
-                ret = new CMasterActivity(this);
-                break;
             case TAKsplit:
-                ret = new CMasterActivity(this, spillingActivityStatistics);
+                ret = new CMasterActivity(this);
                 break;
             case TAKsoap_rowdataset:
             case TAKsoap_rowaction:

diff --git a/thorlcr/thorutil/thbuf.cpp b/thorlcr/thorutil/thbuf.cpp
@@ -2553,7 +2553,7 @@ class CSharedFullSpillingWriteAhead : public CInterfaceOf<ISharedRowStreamReader
     explicit CSharedFullSpillingWriteAhead(CActivityBase *_activity, unsigned _numOutputs, IRowStream *_input, bool _inputGrouped, const SharedRowStreamReaderOptions &_options, IThorRowInterfaces *rowIf, const char *_baseTmpFilename, ICompressHandler *_compressHandler)
         : activity(*_activity), numOutputs(_numOutputs), input(_input), inputGrouped(_inputGrouped), options(_options), compressHandler(_compressHandler), baseTmpFilename(_baseTmpFilename),
         meta(rowIf->queryRowMetaData()), serializer(rowIf->queryRowSerializer()), allocator(rowIf->queryRowAllocator()), deserializer(rowIf->queryRowDeserializer()),
-        inactiveStats(spillingWriteAheadStatistics), previousFileStats(spillingWriteAheadStatistics)
+        inactiveStats(spillStatistics), previousFileStats(spillStatistics)
     {
         assertex(input);
 

diff --git a/thorlcr/thorutil/thormisc.cpp b/thorlcr/thorutil/thormisc.cpp
@@ -74,8 +74,9 @@ static Owned<IMPtagAllocator> ClusterMPAllocator;
 
 // stat. mappings shared between master and slave activities
 const StatisticsMapping spillStatistics({StTimeSpillElapsed, StTimeSortElapsed, StNumSpills, StSizeSpillFile, StSizePeakTempDisk});
+const StatisticsMapping executeStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked});
 const StatisticsMapping soapcallStatistics({StTimeSoapcall});
-const StatisticsMapping basicActivityStatistics({StTimeTotalExecute, StTimeLocalExecute, StTimeBlocked});
+const StatisticsMapping basicActivityStatistics({}, executeStatistics, spillStatistics);
 const StatisticsMapping groupActivityStatistics({StNumGroups, StNumGroupMax}, basicActivityStatistics);
 const StatisticsMapping indexReadFileStatistics({}, diskReadRemoteStatistics, jhtreeCacheStatistics);
 const StatisticsMapping indexReadActivityStatistics({StNumRowsProcessed}, indexReadFileStatistics, basicActivityStatistics);
@@ -89,15 +90,13 @@ const StatisticsMapping joinActivityStatistics({StNumLeftRows, StNumRightRows},
 const StatisticsMapping diskReadActivityStatistics({StNumDiskRowsRead, }, basicActivityStatistics, diskReadRemoteStatistics);
 const StatisticsMapping diskWriteActivityStatistics({StPerReplicated}, basicActivityStatistics, diskWriteRemoteStatistics);
 const StatisticsMapping sortActivityStatistics({}, basicActivityStatistics, spillStatistics);
-const StatisticsMapping graphStatistics({StNumExecutions, StSizeSpillFile, StSizeGraphSpill, StSizePeakTempDisk, StSizePeakEphemeralDisk, StTimeUser, StTimeSystem, StNumContextSwitches, StSizeMemory, StSizePeakMemory, StSizeRowMemory, StSizePeakRowMemory}, basicActivityStatistics);
 const StatisticsMapping diskReadPartStatistics({StNumDiskRowsRead}, diskReadRemoteStatistics);
 const StatisticsMapping indexDistribActivityStatistics({}, basicActivityStatistics, jhtreeCacheStatistics);
 const StatisticsMapping soapcallActivityStatistics({}, basicActivityStatistics, soapcallStatistics);
 const StatisticsMapping hashDedupActivityStatistics({}, spillStatistics, diskWriteRemoteStatistics, basicActivityStatistics);
 const StatisticsMapping hashDistribActivityStatistics({StNumLocalRows, StNumRemoteRows, StSizeRemoteWrite}, basicActivityStatistics);
-const StatisticsMapping spillingActivityStatistics({}, spillStatistics, basicActivityStatistics);
-const StatisticsMapping spillingWriteAheadStatistics({}, spillStatistics);
-const StatisticsMapping loopActivityStatistics({StNumIterations}, spillingActivityStatistics);
+const StatisticsMapping loopActivityStatistics({StNumIterations}, basicActivityStatistics);
+const StatisticsMapping graphStatistics({StNumExecutions, StSizeSpillFile, StSizeGraphSpill, StSizePeakTempDisk, StSizePeakEphemeralDisk, StTimeUser, StTimeSystem, StNumContextSwitches, StSizeMemory, StSizePeakMemory, StSizeRowMemory, StSizePeakRowMemory}, executeStatistics);
 
 const StatKindMap diskToTempStatsMap
 ={ {StSizeDiskWrite, StSizeSpillFile},

diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp
@@ -167,8 +167,6 @@ extern graph_decl const StatisticsMapping soapcallActivityStatistics;
 extern graph_decl const StatisticsMapping indexReadFileStatistics;
 extern graph_decl const StatisticsMapping hashDedupActivityStatistics;
 extern graph_decl const StatisticsMapping hashDistribActivityStatistics;
-extern graph_decl const StatisticsMapping spillingActivityStatistics;
-extern graph_decl const StatisticsMapping spillingWriteAheadStatistics;
 
 // Maps disk related stats to spill stats
 extern graph_decl const std::map<StatisticKind, StatisticKind> diskToTempStatsMap;