HPCC-32241 Temp size for sort to use disk size and track actual graph temp disk usage

thorlcr/graph/thgraph.hpp

@@ -630,7 +630,7 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults
     CChildGraphTable childGraphsTable;
     CGraphStubArrayCopy orderedChildGraphs;
     Owned<IGraphTempHandler> tmpHandler;
-
+    Owned<CFileSizeTracker> tempFileSizeTracker;
     void clean();
 
 protected:
@@ -805,7 +805,24 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults
     virtual void end();
     virtual void abort(IException *e) override;
     virtual IThorGraphResults *createThorGraphResults(unsigned num);
-
+    CFileSizeTracker * queryTempFileSizeTracker()
+    {
+        if (!tempFileSizeTracker)
+            tempFileSizeTracker.setown(new CFileSizeTracker);
+        return tempFileSizeTracker;
+    }
+    offset_t queryPeakTempSize()
+    {
+        if (tempFileSizeTracker)
+            return tempFileSizeTracker->queryPeakSize();
+        return 0;
+    }
+    offset_t queryActiveTempSize()
+    {
+        if (tempFileSizeTracker)
+            return tempFileSizeTracker->queryActiveSize();
+        return 0;
+    }
 // IExceptionHandler
     virtual bool fireException(IException *e);
 
@@ -1175,7 +1192,7 @@ class graph_decl CActivityBase : implements CInterfaceOf<IThorRowInterfaces>, im
     CFileSizeTracker * queryTempFileSizeTracker()
     {
         if (!tempFileSizeTracker)
-            tempFileSizeTracker.setown(new CFileSizeTracker);
+            tempFileSizeTracker.setown(new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker()));
         return tempFileSizeTracker;
     }
     offset_t queryActiveTempSize() const

thorlcr/graph/thgraphslave.cpp

@@ -1323,20 +1323,7 @@ bool CSlaveGraph::serializeStats(MemoryBuffer &mb)
     if (tempHandler)
         stats.mergeStatistic(StSizeGraphSpill, sizeGraphSpill);
 
-    // calculate peak spill size
-    if (started&&initialized)
-    {
-        offset_t activeTempSize = 0;
-        Owned<IThorActivityIterator> iter = getConnectedIterator();
-        ForEach (*iter)
-        {
-            CGraphElementBase &element = iter->query();
-            CSlaveActivity &activity = (CSlaveActivity &)*element.queryActivity();
-            activeTempSize += activity.queryActiveTempSize();
-        }
-        if (activeTempSize > peakTempSize)
-            peakTempSize = activeTempSize;
-    }
+    offset_t peakTempSize = queryPeakTempSize();
     if (peakTempSize)
         stats.mergeStatistic(StSizePeakTempDisk, peakTempSize);
     if (peakTempSize + sizeGraphSpill)

thorlcr/graph/thgraphslave.hpp

@@ -522,7 +522,6 @@ class graphslave_decl CSlaveGraph : public CGraphBase
     bool doneInit = false;
     std::atomic_bool progressActive;
     ProcessInfo processStartInfo;
-    offset_t peakTempSize = 0;
 
 public:
 

thorlcr/msort/tsorts.cpp

@@ -257,8 +257,8 @@ class CWriteIntercept : public CSimpleInterface
         }
         output->flush();
         offset_t end = output->getPosition();
+        dataFile->noteSize(output->getStatistic(StSizeDiskWrite));
         output.clear();
-        dataFile->noteSize(end);
         writeidxofs(end);
         if (idxFileIO)
         {

thorlcr/thorutil/thormisc.hpp

@@ -326,19 +326,29 @@ class CFileSizeTracker: public CInterface
 {
     RelaxedAtomic<offset_t> activeSize{0};
     RelaxedAtomic<offset_t> peakSize{0};
+    CFileSizeTracker * parentFileSizeTracker;
 public:
+    CFileSizeTracker(CFileSizeTracker *parent=nullptr): parentFileSizeTracker(parent)
+    {
+    }
     void growSize(offset_t size)
     {
         if (size)
         {
             offset_t newActiveSize = activeSize.add_fetch(size);
             peakSize.store_max(newActiveSize);
+            if (parentFileSizeTracker)
+                parentFileSizeTracker->growSize(size);
         }
     }
     void shrinkSize(offset_t size)
     {
         if (size)
+        {
             activeSize.fetch_sub(size);
+            if (parentFileSizeTracker)
+                parentFileSizeTracker->shrinkSize(size);
+        }
     }
     offset_t queryActiveSize() const
     {