Merge pull request #18283 from richardkchapman/HPCC-31178

HPCC-31178 Roxie logging too much info - Additional data size ... Reviewed-by: Mark Kelly <[email protected]> Reviewed-by: Gavin Halliday <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Feb 9, 2024 · 21f4d0e · 21f4d0e
2 parents a98ef81 + 9abd956
commit 21f4d0e
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 5 deletions.
diff --git a/roxie/ccd/ccdserver.cpp b/roxie/ccd/ccdserver.cpp
@@ -475,7 +475,8 @@ static const StatisticsMapping keyedJoinStatistics({ StNumServerCacheHits, StNum
                                                     StCycleBlobFetchCycles, StCycleLeafFetchCycles, StCycleNodeFetchCycles, StTimeBlobFetch, StTimeLeafFetch, StTimeNodeFetch,
                                                     StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked,
                                                     StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches,
-                                                    StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries }, joinStatistics);
+                                                    StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries,
+                                                    StSizeContinuationData, StNumContinuationRequests }, joinStatistics);
 static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndexSeeks, StNumIndexScans, StNumIndexWildSeeks,
                                                 StNumIndexSkips, StNumIndexNullSkips, StNumIndexMerges, StNumIndexMergeCompares,
                                                 StNumPreFiltered, StNumPostFiltered, StNumIndexAccepted, StNumIndexRejected,
@@ -486,9 +487,11 @@ static const StatisticsMapping indexStatistics({StNumServerCacheHits, StNumIndex
                                                 StCycleBlobFetchCycles, StCycleLeafFetchCycles, StCycleNodeFetchCycles, StTimeBlobFetch, StTimeLeafFetch, StTimeNodeFetch,
                                                 StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked,
                                                 StNumNodeDiskFetches, StNumLeafDiskFetches, StNumBlobDiskFetches,
-                                                StNumIndexRowsRead, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries }, actStatistics);
+                                                StNumIndexRowsRead, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries,
+                                                StSizeContinuationData, StNumContinuationRequests }, actStatistics);
 static const StatisticsMapping diskStatistics({StNumServerCacheHits, StNumDiskRowsRead, StNumDiskSeeks, StNumDiskAccepted,
-                                               StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries }, actStatistics);
+                                               StNumDiskRejected, StSizeAgentReply, StTimeAgentWait, StTimeAgentQueue, StTimeAgentProcess, StTimeIBYTIDelay, StNumAckRetries,
+                                               StSizeContinuationData, StNumContinuationRequests }, actStatistics);
 static const StatisticsMapping soapStatistics({ StTimeSoapcall }, actStatistics);
 static const StatisticsMapping groupStatistics({ StNumGroups, StNumGroupMax }, actStatistics);
 static const StatisticsMapping sortStatistics({ StTimeSortElapsed }, actStatistics);
@@ -519,7 +522,7 @@ extern const StatisticsMapping accumulatedStatistics({StWhenFirstRow, StTimeLoca
                                                       StNumSocketWrites, StSizeSocketWrite, StTimeSocketWriteIO,
                                                       StNumSocketReads, StSizeSocketRead, StTimeSocketReadIO,
                                                       StCycleIndexCacheBlockedCycles, StTimeIndexCacheBlocked,
-                                                      StNumAckRetries,
+                                                      StNumAckRetries, StSizeContinuationData, StNumContinuationRequests
                                                       });
 
 //=================================================================================
@@ -5246,7 +5249,9 @@ class CRemoteResultAdaptor : implements IEngineRowStream, implements IFinalRoxie
                             // We got back first chunk but there is more.
                             // resend the packet, with the cursor info provided.
                             // MORE - if smart-stepping, we don't want to send the continuation immediately. Other cases it's not clear that we do.
-                            if (ctxTraceLevel > 1)
+                            activity.noteStatistic(StNumContinuationRequests, 1);
+                            activity.noteStatistic(StSizeContinuationData, metaLen);
+                            if (doTrace(traceRoxiePackets))
                             {
                                 StringBuffer s;
                                 activity.queryLogCtx().CTXLOG("Additional data size %d on query %s mergeOrder %p", metaLen, header.toString(s).str(), mergeOrder);

diff --git a/system/jlib/jstatcodes.h b/system/jlib/jstatcodes.h
@@ -296,6 +296,8 @@ enum StatisticKind
     StTimeAgentProcess,
     StCycleAgentProcessCycles,
     StNumAckRetries,
+    StSizeContinuationData,
+    StNumContinuationRequests,
     StMax,
 
     //For any quantity there is potentially the following variants.

diff --git a/system/jlib/jstats.cpp b/system/jlib/jstats.cpp
@@ -957,6 +957,8 @@ static const constexpr StatisticMeta statsMetaData[StMax] = {
     { TIMESTAT(AgentProcess) },
     { CYCLESTAT(AgentProcess) },
     { NUMSTAT(AckRetries) },
+    { SIZESTAT(ContinuationData) },
+    { NUMSTAT(ContinuationRequests) },
 };
 
 static MapStringTo<StatisticKind, StatisticKind> statisticNameMap(true);