From 226c107da5c565dde457b4dbeb3d767f888d5d93 Mon Sep 17 00:00:00 2001 From: Gavin Halliday Date: Thu, 3 Oct 2024 11:31:58 +0100 Subject: [PATCH] HPCC-32764 Periodically log information about cpu-throttling Signed-off-by: Gavin Halliday --- system/jlib/jdebug.cpp | 112 ++++++++++++++++++++++++++++++-- system/jlib/jdebug.hpp | 7 ++ system/jlib/jstring.cpp | 34 ++++++++++ system/jlib/jstring.hpp | 18 ++++- testing/unittests/jlibtests.cpp | 17 +++++ 5 files changed, 181 insertions(+), 7 deletions(-) diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp index 4dcc7a141cf..5fa81ba1b96 100644 --- a/system/jlib/jdebug.cpp +++ b/system/jlib/jdebug.cpp @@ -816,6 +816,54 @@ static struct CNtKernelInformation } NtKernelFunctions; #endif +//=========================================================================== + +#ifndef _WIN32 + +static std::atomic gatheredGroup{false}; +static StringAttr cgroup; +static CriticalSection csgroupCs; +static const char * queryCGroup() +{ + if (!gatheredGroup) + { + CriticalBlock block(csgroupCs); + if (!gatheredGroup) + { + StringBuffer contents; + if (loadBinaryFile(contents, "/proc/self/cgroup", false)) + { + auto processLine = [](size_t len, const char * ln) + { + //Note ln points at the start of the line, but the line is not null terminated + switch (*ln) + { + case '0': + if (strncmp(ln, "0::/", 4) == 0) + { + //Format is 0::/ + //If not running in a container the "cgroup" may be something like user.slice/user-1000.slice/user@1000.service/.... + //If so ignore because it is not a real cgroup + if (!memchr(ln+4, '/', len-4)) + cgroup.set(ln+4, len-4); + } + break; + } + //Some systems with version 1 cgroups have :cpu,cpuacct:/ + const char * match = (const char *)jmemmem(len, ln, 14, ":cpu,cpuacct:/"); + if (match) + cgroup.set(match+14, (ln + len) - (match + 14)); + }; + + processLines(contents, processLine); + } + gatheredGroup = true; + } + } + return cgroup.get(); +} + +#endif //=========================================================================== @@ -893,6 +941,9 @@ SystemProcessInfo SystemProcessInfo::operator - (const SystemProcessInfo & rhs) result.activeDataMemory = activeDataMemory - rhs.activeDataMemory; result.majorFaults = majorFaults - rhs.majorFaults; result.numThreads = numThreads - rhs.numThreads; + result.numPeriods = numPeriods - rhs.numPeriods; + result.numThrottledPeriods = numThrottledPeriods - rhs.numThrottledPeriods; + result.timeThrottledNs = timeThrottledNs - rhs.timeThrottledNs; return result; } @@ -995,7 +1046,7 @@ bool ProcessInfo::update(unsigned flags) if (loadBinaryFile(contents, "/proc/self/status", false)) { contextSwitches = 0; - auto processLine = [this](const char * cur) + auto processLine = [this](size_t len, const char * cur) { __uint64 value; switch (*cur) @@ -1070,7 +1121,7 @@ bool SystemInfo::update(unsigned flags) StringBuffer contents; if (loadBinaryFile(contents, "/proc/stat", false)) { - auto processLine = [this](const char * ln) + auto processLine = [this](size_t len, const char * ln) { switch (*ln) { @@ -1105,6 +1156,52 @@ bool SystemInfo::update(unsigned flags) processLines(contents, processLine); } + auto processLine = [this](size_t len, const char * ln) + { + switch (*ln) + { + case 'n': + if (strncmp(ln, "nr_periods ", 11) == 0) + numPeriods = strtod(ln+11, nullptr); + else if (strncmp(ln, "nr_throttled ", 13) == 0) + numThrottledPeriods = strtod(ln+13, nullptr); + break; + case 't': + if (strncmp(ln, "throttled_usec ", 15) == 0) + timeThrottledNs = strtod(ln+15, nullptr) * 1000; + else if (strncmp(ln, "throttled_time ", 15) == 0) + timeThrottledNs = strtod(ln+15, nullptr); + break; + } + }; + + bool done = false; + const char * cgroup = queryCGroup(); + if (cgroup) + { + //Version 2 of cgroups has the information in cgroup/ + VStringBuffer filename("/sys/fs/cgroup/%s/cpu.stat", cgroup); + if (loadBinaryFile(contents.clear(), filename.str(), false)) + { + processLines(contents, processLine); + done = true; + } + else + { + //Some systems with version 1 cgroups have the information in /sys/fs/cgroup/cpu//cpu.stat + filename.clear().appendf("/sys/fs/cgroup/cpu/%s/cpu.stat", cgroup); + if (loadBinaryFile(contents.clear(), filename.str(), false)) + { + processLines(contents, processLine); + done = true; + } + } + } + + //If the version 2 file was not found look for ther version 1 information in cgroup/cpu + if (!done && loadBinaryFile(contents.clear(), "/sys/fs/cgroup/cpu/cpu.stat", false)) + processLines(contents, processLine); + return true; #endif } @@ -2502,9 +2599,16 @@ class CExtendedStats // Disk network and cpu stats } if (totalcpu) { - if (out.length()&&(out.charAt(out.length()-1)!=' ')) - out.append(' '); + ensureSeparator(out, ' '); out.appendf("CPU: usr=%d sys=%d iow=%d idle=%d", deltacpu.getUserPercent(), deltacpu.getSystemPercent(), deltacpu.getIoWaitPercent(), deltacpu.getIdlePercent()); + + __uint64 periods = deltacpu.getNumPeriods(); + if (periods) + { + unsigned throttling = deltacpu.getNumThrottledPeriods() * 100 / periods; + __uint64 timeThrottledNs = deltacpu.getTimeThrottledNs(); + out.appendf(" thr=%u%% thrns=%llu", throttling, timeThrottledNs); + } } return true; } diff --git a/system/jlib/jdebug.hpp b/system/jlib/jdebug.hpp index 0a8a0dacfcc..a4d5d41c705 100644 --- a/system/jlib/jdebug.hpp +++ b/system/jlib/jdebug.hpp @@ -446,6 +446,10 @@ class jlib_decl SystemProcessInfo unsigned getSystemPercent() const; unsigned getUserPercent() const; + __uint64 getNumPeriods() const { return numPeriods; } + __uint64 getNumThrottledPeriods() const { return numThrottledPeriods; } + __uint64 getTimeThrottledNs() const { return timeThrottledNs; } + __uint64 getTotal() const { return user + system + idle + iowait; } protected: __uint64 user = 0; // user time in jiffies (~`1/100s) @@ -461,6 +465,9 @@ class jlib_decl SystemProcessInfo __uint64 activeDataMemory = 0; __uint64 majorFaults = 0; __uint64 numThreads = 0; + __uint64 numPeriods = 0; + __uint64 numThrottledPeriods = 0; + __uint64 timeThrottledNs = 0; }; class jlib_decl ProcessInfo : public SystemProcessInfo diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp index 50951938ae5..7e581f2503a 100644 --- a/system/jlib/jstring.cpp +++ b/system/jlib/jstring.cpp @@ -2902,6 +2902,12 @@ void getSnakeCase(StringBuffer & out, const char * camelValue) } } +void ensureSeparator(StringBuffer & out, char separator) +{ + if (out.length() && (out.charAt(out.length()-1) != separator)) + out.append(separator); +} + /** * stristr - Case insensitive strstr() * @haystack: Where we will search for our @needle @@ -2949,3 +2955,31 @@ const char * stristr (const char *haystack, const char *needle) } return nullptr; } + + +const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle) +{ + if (lenNeedle == 0) + return haystack; + + if (lenHaystack < lenNeedle) + return nullptr; + + const char * search = (const char *)needle; + char first = *search; + if (lenNeedle == 1) + return memchr(haystack, first, lenHaystack); + + const char * buffer = (const char *)haystack; + for (size_t i = 0; i <= lenHaystack - lenNeedle; i++) + { + //Special case the first character to avoid a function call each iteration. + if (buffer[i] == first) + { + if (memcmp(buffer + i + 1, search + 1, lenNeedle-1) == 0) + return buffer + i; + } + } + + return nullptr; +} diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp index b3fe7651daf..d61b7fb7a1f 100644 --- a/system/jlib/jstring.hpp +++ b/system/jlib/jstring.hpp @@ -632,11 +632,18 @@ void processLines(const StringBuffer & content, LineProcessor process) const char * cur = content; while (*cur) { - process(cur); const char * next = strchr(cur, '\n'); - if (!next) + if (next) + { + if (next != cur) + process(next-cur, cur); + cur = next+1; + } + else + { + process(strlen(cur), cur); break; - cur = next+1; + } } } @@ -646,5 +653,10 @@ extern jlib_decl void processOptionString(const char * options, optionCallback c extern jlib_decl const char * stristr(const char *haystack, const char *needle); extern jlib_decl void getSnakeCase(StringBuffer & out, const char * camelValue); +//If the string has any characters, ensure the last character matches the separator +extern jlib_decl void ensureSeparator(StringBuffer & out, char separator); + +//Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own +extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle); #endif diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp index ddf01c08461..398077ac3f4 100644 --- a/testing/unittests/jlibtests.cpp +++ b/testing/unittests/jlibtests.cpp @@ -3390,6 +3390,7 @@ class MachineInfoTimingTest : public CppUnit::TestFixture DBGLOG(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)", (unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000), (unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches()); + DBGLOG(" Throttled: Periods(%llu/%llu) TimeNs(%llu)", deltaSystem.getNumThrottledPeriods(), deltaSystem.getNumPeriods(), deltaSystem.getTimeThrottledNs()); } for (unsigned j=0; j < i*100000000; j++) @@ -4630,6 +4631,7 @@ class JLibStringTest : public CppUnit::TestFixture public: CPPUNIT_TEST_SUITE(JLibStringTest); CPPUNIT_TEST(testStristr); + CPPUNIT_TEST(testMemMem); CPPUNIT_TEST_SUITE_END(); void testStristr() @@ -4647,6 +4649,21 @@ class JLibStringTest : public CppUnit::TestFixture CPPUNIT_ASSERT_EQUAL_STR(stristr("", "ABC"), ""); CPPUNIT_ASSERT_EQUAL_STR(stristr("ABC", ""), ""); } + + void testMemMem() + { + constexpr const char * haystack = "abcdefghijklmnopqrstuvwxyz"; + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 0, nullptr)); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(3, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(2, haystack, 3, "abc")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+7), jmemmem(10, haystack, 3, "hij")); + CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(10, haystack, 3, "ijk")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 1, "i")); + CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(8, haystack, 1, "i")); + CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(9, haystack, 2, "ij")); + CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 2, "ij")); + } }; CPPUNIT_TEST_SUITE_REGISTRATION( JLibStringTest );