Merge pull request #19186 from ghalliday/issue32764

HPCC-32764 Periodically log information about cpu-throttling Reviewed-by: Mark Kelly [email protected] Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · Oct 14, 2024 · 6e4a463 · 6e4a463
2 parents 7461327 + 226c107
commit 6e4a463
Show file tree

Hide file tree

Showing 5 changed files with 181 additions and 7 deletions.
diff --git a/system/jlib/jdebug.cpp b/system/jlib/jdebug.cpp
@@ -816,6 +816,54 @@ static struct CNtKernelInformation
 } NtKernelFunctions;
 #endif
 
+//===========================================================================
+
+#ifndef _WIN32
+
+static std::atomic<bool> gatheredGroup{false};
+static StringAttr cgroup;
+static CriticalSection csgroupCs;
+static const char * queryCGroup()
+{
+    if (!gatheredGroup)
+    {
+        CriticalBlock block(csgroupCs);
+        if (!gatheredGroup)
+        {
+            StringBuffer contents;
+            if (loadBinaryFile(contents, "/proc/self/cgroup", false))
+            {
+                auto processLine = [](size_t len, const char * ln)
+                {
+                    //Note ln points at the start of the line, but the line is not null terminated
+                    switch (*ln)
+                    {
+                    case '0':
+                        if (strncmp(ln, "0::/", 4) == 0)
+                        {
+                            //Format is 0::/<cgroup>
+                            //If not running in a container the "cgroup" may be something like user.slice/user-1000.slice/[email protected]/....
+                            //If so ignore because it is not a real cgroup
+                            if (!memchr(ln+4, '/', len-4))
+                                cgroup.set(ln+4, len-4);
+                        }
+                        break;
+                    }
+                    //Some systems with version 1 cgroups have <n>:cpu,cpuacct:/<cgroup>
+                    const char  * match = (const char *)jmemmem(len, ln, 14, ":cpu,cpuacct:/");
+                    if (match)
+                        cgroup.set(match+14, (ln + len) - (match + 14));
+                };
+
+                processLines(contents, processLine);
+            }
+            gatheredGroup = true;
+        }
+    }
+    return cgroup.get();
+}
+
+#endif
 
 //===========================================================================
 
@@ -893,6 +941,9 @@ SystemProcessInfo SystemProcessInfo::operator - (const SystemProcessInfo & rhs)
     result.activeDataMemory = activeDataMemory - rhs.activeDataMemory;
     result.majorFaults = majorFaults - rhs.majorFaults;
     result.numThreads = numThreads - rhs.numThreads;
+    result.numPeriods = numPeriods - rhs.numPeriods;
+    result.numThrottledPeriods = numThrottledPeriods - rhs.numThrottledPeriods;
+    result.timeThrottledNs = timeThrottledNs - rhs.timeThrottledNs;
     return result;
 }
 
@@ -995,7 +1046,7 @@ bool ProcessInfo::update(unsigned flags)
         if (loadBinaryFile(contents, "/proc/self/status", false))
         {
             contextSwitches = 0;
-            auto processLine = [this](const char * cur)
+            auto processLine = [this](size_t len, const char * cur)
             {
                 __uint64 value;
                 switch (*cur)
@@ -1070,7 +1121,7 @@ bool SystemInfo::update(unsigned flags)
     StringBuffer contents;
     if (loadBinaryFile(contents, "/proc/stat", false))
     {
-        auto processLine = [this](const char * ln)
+        auto processLine = [this](size_t len, const char * ln)
         {
             switch (*ln)
             {
@@ -1105,6 +1156,52 @@ bool SystemInfo::update(unsigned flags)
         processLines(contents, processLine);
     }
 
+    auto processLine = [this](size_t len, const char * ln)
+    {
+        switch (*ln)
+        {
+        case 'n':
+            if (strncmp(ln, "nr_periods ", 11) == 0)
+                numPeriods = strtod(ln+11, nullptr);
+            else if (strncmp(ln, "nr_throttled ", 13) == 0)
+                numThrottledPeriods = strtod(ln+13, nullptr);
+            break;
+        case 't':
+            if (strncmp(ln, "throttled_usec ", 15) == 0)
+                timeThrottledNs = strtod(ln+15, nullptr) * 1000;
+            else if (strncmp(ln, "throttled_time ", 15) == 0)
+                timeThrottledNs = strtod(ln+15, nullptr);
+            break;
+        }
+    };
+
+    bool done = false;
+    const char * cgroup = queryCGroup();
+    if (cgroup)
+    {
+        //Version 2 of cgroups has the information in cgroup/<cgroup>
+        VStringBuffer filename("/sys/fs/cgroup/%s/cpu.stat", cgroup);
+        if (loadBinaryFile(contents.clear(), filename.str(), false))
+        {
+            processLines(contents, processLine);
+            done = true;
+        }
+        else
+        {
+            //Some systems with version 1 cgroups have the information in /sys/fs/cgroup/cpu/<cgroup>/cpu.stat
+            filename.clear().appendf("/sys/fs/cgroup/cpu/%s/cpu.stat", cgroup);
+            if (loadBinaryFile(contents.clear(), filename.str(), false))
+            {
+                processLines(contents, processLine);
+                done = true;
+            }
+        }
+    }
+
+    //If the version 2 file was not found look for ther version 1 information in cgroup/cpu
+    if (!done && loadBinaryFile(contents.clear(), "/sys/fs/cgroup/cpu/cpu.stat", false))
+        processLines(contents, processLine);
+
     return true;
 #endif
 }
@@ -2502,9 +2599,16 @@ class CExtendedStats  // Disk network and cpu stats
         }
         if (totalcpu)
         {
-            if (out.length()&&(out.charAt(out.length()-1)!=' '))
-                out.append(' ');
+            ensureSeparator(out, ' ');
             out.appendf("CPU: usr=%d sys=%d iow=%d idle=%d", deltacpu.getUserPercent(), deltacpu.getSystemPercent(), deltacpu.getIoWaitPercent(), deltacpu.getIdlePercent());
+
+            __uint64 periods = deltacpu.getNumPeriods();
+            if (periods)
+            {
+                unsigned throttling = deltacpu.getNumThrottledPeriods() * 100 / periods;
+                __uint64 timeThrottledNs = deltacpu.getTimeThrottledNs();
+                out.appendf(" thr=%u%% thrns=%llu", throttling, timeThrottledNs);
+            }
         }
         return true;
     }

diff --git a/system/jlib/jdebug.hpp b/system/jlib/jdebug.hpp
@@ -446,6 +446,10 @@ class jlib_decl SystemProcessInfo
     unsigned getSystemPercent() const;
     unsigned getUserPercent() const;
 
+    __uint64 getNumPeriods() const { return numPeriods; }
+    __uint64 getNumThrottledPeriods() const { return numThrottledPeriods; }
+    __uint64 getTimeThrottledNs() const { return timeThrottledNs; }
+
     __uint64 getTotal() const { return user + system + idle + iowait; }
 protected:
     __uint64 user = 0;      // user time in jiffies (~`1/100s)
@@ -461,6 +465,9 @@ class jlib_decl SystemProcessInfo
     __uint64 activeDataMemory = 0;
     __uint64 majorFaults = 0;
     __uint64 numThreads = 0;
+    __uint64 numPeriods = 0;
+    __uint64 numThrottledPeriods = 0;
+    __uint64 timeThrottledNs = 0;
 };
 
 class jlib_decl ProcessInfo : public SystemProcessInfo

diff --git a/system/jlib/jstring.cpp b/system/jlib/jstring.cpp
@@ -2902,6 +2902,12 @@ void getSnakeCase(StringBuffer & out, const char * camelValue)
     }
 }
 
+void ensureSeparator(StringBuffer & out, char separator)
+{
+    if (out.length() && (out.charAt(out.length()-1) != separator))
+        out.append(separator);
+}
+
 /**
  * stristr - Case insensitive strstr()
  * @haystack: Where we will search for our @needle
@@ -2949,3 +2955,31 @@ const char * stristr (const char *haystack, const char *needle)
     }
     return nullptr;
 }
+
+
+const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle)
+{
+    if (lenNeedle == 0)
+        return haystack;
+
+    if (lenHaystack < lenNeedle)
+        return nullptr;
+
+    const char * search = (const char *)needle;
+    char first = *search;
+    if (lenNeedle == 1)
+        return memchr(haystack, first, lenHaystack);
+
+    const char * buffer = (const char *)haystack;
+    for (size_t i = 0; i <= lenHaystack - lenNeedle; i++)
+    {
+        //Special case the first character to avoid a function call each iteration.
+        if (buffer[i] == first)
+        {
+            if (memcmp(buffer + i + 1, search + 1, lenNeedle-1) == 0)
+                return buffer + i;
+        }
+    }
+
+    return nullptr;
+}
diff --git a/system/jlib/jstring.hpp b/system/jlib/jstring.hpp
@@ -632,11 +632,18 @@ void processLines(const StringBuffer & content, LineProcessor process)
     const char * cur = content;
     while (*cur)
     {
-        process(cur);
         const char * next = strchr(cur, '\n');
-        if (!next)
+        if (next)
+        {
+            if (next != cur)
+                process(next-cur, cur);
+            cur = next+1;
+        }
+        else
+        {
+            process(strlen(cur), cur);
             break;
-        cur = next+1;
+        }
     }
 }
 
@@ -646,5 +653,10 @@ extern jlib_decl void processOptionString(const char * options, optionCallback c
 
 extern jlib_decl const char * stristr(const char *haystack, const char *needle);
 extern jlib_decl void getSnakeCase(StringBuffer & out, const char * camelValue);
+//If the string has any characters, ensure the last character matches the separator
+extern jlib_decl void ensureSeparator(StringBuffer & out, char separator);
+
+//Search for one block of bytes within another block of bytes - memmem is not standard, so we provide our own
+extern jlib_decl const void * jmemmem(size_t lenHaystack, const void * haystack, size_t lenNeedle, const void *needle);
 
 #endif
diff --git a/testing/unittests/jlibtests.cpp b/testing/unittests/jlibtests.cpp
@@ -3390,6 +3390,7 @@ class MachineInfoTimingTest : public CppUnit::TestFixture
                 DBGLOG(" Process: User(%u) System(%u) Total(%u) %u%% Ctx(%" I64F "u)",
                         (unsigned)(deltaProcess.getUserNs() / 1000000), (unsigned)(deltaProcess.getSystemNs() / 1000000), (unsigned)(deltaProcess.getTotalNs() / 1000000),
                         (unsigned)((deltaProcess.getUserNs() * 100) / deltaSystem.getTotalNs()), deltaProcess.getNumContextSwitches());
+                DBGLOG(" Throttled: Periods(%llu/%llu) TimeNs(%llu)", deltaSystem.getNumThrottledPeriods(), deltaSystem.getNumPeriods(), deltaSystem.getTimeThrottledNs());
             }
 
             for (unsigned j=0; j < i*100000000; j++)
@@ -4630,6 +4631,7 @@ class JLibStringTest : public CppUnit::TestFixture
 public:
     CPPUNIT_TEST_SUITE(JLibStringTest);
         CPPUNIT_TEST(testStristr);
+        CPPUNIT_TEST(testMemMem);
     CPPUNIT_TEST_SUITE_END();
 
     void testStristr()
@@ -4647,6 +4649,21 @@ class JLibStringTest : public CppUnit::TestFixture
         CPPUNIT_ASSERT_EQUAL_STR(stristr("", "ABC"), "");
         CPPUNIT_ASSERT_EQUAL_STR(stristr("ABC", ""), "");
     }
+
+    void testMemMem()
+    {
+        constexpr const char * haystack = "abcdefghijklmnopqrstuvwxyz";
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 0, nullptr));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(10, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack), jmemmem(3, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(2, haystack, 3, "abc"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+7), jmemmem(10, haystack, 3, "hij"));
+        CPPUNIT_ASSERT_EQUAL((const void*)nullptr, jmemmem(10, haystack, 3, "ijk"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 1, "i"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(8, haystack, 1, "i"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(nullptr), jmemmem(9, haystack, 2, "ij"));
+        CPPUNIT_ASSERT_EQUAL((const void*)(haystack+8), jmemmem(10, haystack, 2, "ij"));
+    }
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( JLibStringTest );