diff --git a/thorlcr/graph/thgraph.hpp b/thorlcr/graph/thgraph.hpp index b0adfbade47..b1045fda17a 100644 --- a/thorlcr/graph/thgraph.hpp +++ b/thorlcr/graph/thgraph.hpp @@ -630,7 +630,7 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults CChildGraphTable childGraphsTable; CGraphStubArrayCopy orderedChildGraphs; Owned tmpHandler; - + Owned tempFileSizeTracker; void clean(); protected: @@ -805,7 +805,24 @@ class graph_decl CGraphBase : public CGraphStub, implements IEclGraphResults virtual void end(); virtual void abort(IException *e) override; virtual IThorGraphResults *createThorGraphResults(unsigned num); - + CFileSizeTracker * queryTempFileSizeTracker() + { + if (!tempFileSizeTracker) + tempFileSizeTracker.setown(new CFileSizeTracker); + return tempFileSizeTracker; + } + offset_t queryPeakTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryPeakSize(); + return 0; + } + offset_t queryActiveTempSize() + { + if (tempFileSizeTracker) + return tempFileSizeTracker->queryActiveSize(); + return 0; + } // IExceptionHandler virtual bool fireException(IException *e); @@ -1175,7 +1192,7 @@ class graph_decl CActivityBase : implements CInterfaceOf, im CFileSizeTracker * queryTempFileSizeTracker() { if (!tempFileSizeTracker) - tempFileSizeTracker.setown(new CFileSizeTracker); + tempFileSizeTracker.setown(new CFileSizeTracker(queryGraph().queryParent()->queryTempFileSizeTracker())); return tempFileSizeTracker; } offset_t queryActiveTempSize() const diff --git a/thorlcr/graph/thgraphslave.cpp b/thorlcr/graph/thgraphslave.cpp index 14c8d7ea6d0..e39c45909d5 100644 --- a/thorlcr/graph/thgraphslave.cpp +++ b/thorlcr/graph/thgraphslave.cpp @@ -1323,20 +1323,7 @@ bool CSlaveGraph::serializeStats(MemoryBuffer &mb) if (tempHandler) stats.mergeStatistic(StSizeGraphSpill, sizeGraphSpill); - // calculate peak spill size - if (started&&initialized) - { - offset_t activeTempSize = 0; - Owned iter = getConnectedIterator(); - ForEach (*iter) - { - CGraphElementBase &element = iter->query(); - CSlaveActivity &activity = (CSlaveActivity &)*element.queryActivity(); - activeTempSize += activity.queryActiveTempSize(); - } - if (activeTempSize > peakTempSize) - peakTempSize = activeTempSize; - } + offset_t peakTempSize = queryPeakTempSize(); if (peakTempSize) stats.mergeStatistic(StSizePeakTempDisk, peakTempSize); if (peakTempSize + sizeGraphSpill) diff --git a/thorlcr/graph/thgraphslave.hpp b/thorlcr/graph/thgraphslave.hpp index f6b73c912c1..1c9595348f2 100644 --- a/thorlcr/graph/thgraphslave.hpp +++ b/thorlcr/graph/thgraphslave.hpp @@ -522,7 +522,6 @@ class graphslave_decl CSlaveGraph : public CGraphBase bool doneInit = false; std::atomic_bool progressActive; ProcessInfo processStartInfo; - offset_t peakTempSize = 0; public: diff --git a/thorlcr/msort/tsorts.cpp b/thorlcr/msort/tsorts.cpp index 55afb16704a..7e58fa50ea1 100644 --- a/thorlcr/msort/tsorts.cpp +++ b/thorlcr/msort/tsorts.cpp @@ -257,8 +257,8 @@ class CWriteIntercept : public CSimpleInterface } output->flush(); offset_t end = output->getPosition(); + dataFile->noteSize(output->getStatistic(StSizeDiskWrite)); output.clear(); - dataFile->noteSize(end); writeidxofs(end); if (idxFileIO) { diff --git a/thorlcr/thorutil/thormisc.hpp b/thorlcr/thorutil/thormisc.hpp index cf0b92bb4b0..5dec5ee43bc 100644 --- a/thorlcr/thorutil/thormisc.hpp +++ b/thorlcr/thorutil/thormisc.hpp @@ -326,19 +326,29 @@ class CFileSizeTracker: public CInterface { RelaxedAtomic activeSize{0}; RelaxedAtomic peakSize{0}; + CFileSizeTracker * parentFileSizeTracker; public: + CFileSizeTracker(CFileSizeTracker *parent=nullptr): parentFileSizeTracker(parent) + { + } void growSize(offset_t size) { if (size) { offset_t newActiveSize = activeSize.add_fetch(size); peakSize.store_max(newActiveSize); + if (parentFileSizeTracker) + parentFileSizeTracker->growSize(size); } } void shrinkSize(offset_t size) { if (size) + { activeSize.fetch_sub(size); + if (parentFileSizeTracker) + parentFileSizeTracker->shrinkSize(size); + } } offset_t queryActiveSize() const {