Merge pull request #20498 from ymanton/disclaim-scc

Add functionality to disclaim SCC memory periodically

runtime/compiler/control/HookedByTheJit.cpp

@@ -4515,14 +4515,26 @@ size_t getRSS_Kb()
    return rss;
    }
 
+#if defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX)
+void disclaimSharedClassCache(TR_J9SharedCache *sharedCache, uint64_t crtElapsedTime)
+   {
+   size_t rssBefore = getRSS_Kb();
+   int32_t numDisclaimed = sharedCache->disclaimSharedCaches();
+   size_t rssAfter = getRSS_Kb();
+   if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d SCC segments  RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%",
+                                     (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
+   }
+#endif // defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX)
+
 void disclaimDataCaches(uint64_t crtElapsedTime)
    {
    size_t rssBefore = getRSS_Kb();
    int numDisclaimed = TR_DataCacheManager::getManager()->disclaimAllDataCaches();
    size_t rssAfter = getRSS_Kb();
    if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
-      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Data Cache segments  RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB",
-                                     (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter);
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Data Cache segments  RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%",
+                                     (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
    }
 
 void disclaimIProfilerSegments(uint64_t crtElapsedTime)
@@ -4535,8 +4547,8 @@ void disclaimIProfilerSegments(uint64_t crtElapsedTime)
       int numSegDisclaimed = iprofilerAllocator->disclaimAllSegments();
       size_t rssAfter = getRSS_Kb();
       if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
-         TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d IProfiler segments out of %d. RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB",
-                                        (uint32_t)crtElapsedTime, numSegDisclaimed, iprofilerAllocator->getNumSegments(), rssBefore, rssAfter, rssBefore - rssAfter);
+         TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d IProfiler segments out of %d. RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%",
+                                        (uint32_t)crtElapsedTime, numSegDisclaimed, iprofilerAllocator->getNumSegments(), rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
       }
    }
 
@@ -4546,7 +4558,7 @@ void disclaimCodeCaches(uint64_t crtElapsedTime)
    int numDisclaimed = TR::CodeCacheManager::instance()->disclaimAllCodeCaches();
    size_t rssAfter = getRSS_Kb();
    if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance))
-      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB = %5.2f%%",
+      TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%",
                                      (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore));
    }
 
@@ -4557,6 +4569,7 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime,
    static uint64_t lastCodeCacheDisclaimTime = 0;
    static int32_t  lastNumAllocatedCodeCaches = 0;
    static uint64_t lastIProfilerDisclaimTime = 0;
+   static uint64_t lastSCCDisclaimTime = 0;
    static uint32_t lastNumCompilationsDuringIProfilerDisclaim = 0;
 
    J9JITConfig *jitConfig = compInfo->getJITConfig();
@@ -4571,6 +4584,20 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime,
    if (javaVM->phase != J9VM_PHASE_NOT_STARTUP || jitState == STARTUP_STATE)
       return;
 
+#if defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX)
+   TR_J9VMBase *fej9 = TR_J9VMBase::get(jitConfig, compInfo->getSamplerThread(), TR_J9VMBase::AOT_VM);
+   TR_J9SharedCache *sharedCache = fej9->sharedCache();
+   if (sharedCache && sharedCache->isDisclaimEnabled())
+      {
+      // Disclaim if there was a large time interval since the last disclaim
+      if (crtElapsedTime > lastSCCDisclaimTime + 12 * TR::Options::_minTimeBetweenMemoryDisclaims)
+         {
+         disclaimSharedClassCache(sharedCache, crtElapsedTime);
+         lastSCCDisclaimTime = crtElapsedTime;
+         }
+      }
+#endif // defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX)
+
    if (TR_DataCacheManager::getManager()->isDisclaimEnabled())
       {
       // Ensure we don't do it too often

runtime/compiler/control/J9Options.cpp

@@ -2929,7 +2929,8 @@ J9::Options::fePostProcessJIT(void * base)
 
    if (!self()->getOption(TR_DisableDataCacheDisclaiming) ||
        !self()->getOption(TR_DisableIProfilerDataDisclaiming) ||
-       self()->getOption(TR_EnableCodeCacheDisclaiming))
+       self()->getOption(TR_EnableCodeCacheDisclaiming) ||
+       self()->getOption(TR_EnableSharedCacheDisclaiming))
       {
       // Check requirements for memory disclaiming (Linux kernel and default page size)
       TR::Options::disableMemoryDisclaimIfNeeded(jitConfig);
@@ -3002,6 +3003,7 @@ J9::Options::disableMemoryDisclaimIfNeeded(J9JITConfig *jitConfig)
       TR::Options::getCmdLineOptions()->setOption(TR_DisableDataCacheDisclaiming);
       TR::Options::getCmdLineOptions()->setOption(TR_DisableIProfilerDataDisclaiming);
       TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false);
+      TR::Options::getCmdLineOptions()->setOption(TR_EnableSharedCacheDisclaiming, false);
       }
    return shouldDisableMemoryDisclaim;
    }

runtime/compiler/control/OptionsPostRestore.cpp

@@ -830,7 +830,8 @@ J9::OptionsPostRestore::postProcessInternalCompilerOptions()
 
    if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) ||
        !TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming) ||
-       TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming))
+       TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming) ||
+       TR::Options::getCmdLineOptions()->getOption(TR_EnableSharedCacheDisclaiming))
       {
       TR::Options::disableMemoryDisclaimIfNeeded(_jitConfig);
       }

runtime/compiler/env/J9SharedCache.cpp

@@ -48,13 +48,26 @@
 #include "runtime/JITServerAOTDeserializer.hpp"
 #endif
 
+// for madvise
+#ifdef LINUX
+#include <sys/mman.h>
+#ifndef MADV_NOHUGEPAGE
+#define MADV_NOHUGEPAGE  15
+#endif // MADV_NOHUGEPAGE
+#ifndef MADV_PAGEOUT
+#define MADV_PAGEOUT     21
+#endif // MADV_PAGEOUT
+#endif
+
 #define LOG(logLevel, format, ...)               \
    if (_logLevel >= logLevel)                    \
       {                                          \
       log("" format "", ##__VA_ARGS__);          \
       }
 
 // From CompositeCache.cpp
+#define RWUPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->readWriteSRP)
+#define CAEND(ca) (((uint8_t *)(ca)) + (ca)->totalBytes)
 #define UPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->updateSRP)
 #define SEGUPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->segmentSRP)
 
@@ -105,6 +118,67 @@ TR_J9SharedCache::validateAOTHeader(J9JITConfig *jitConfig, J9VMThread *vmThread
       }
    }
 
+#if defined(LINUX)
+bool TR_J9SharedCache::disclaim(const uint8_t *start, const uint8_t *end, UDATA pageSize, bool trace)
+   {
+   uint8_t *nextPage = (uint8_t *)(((UDATA)start + (pageSize - 1)) & ~(pageSize - 1));
+   if (nextPage < end)
+      {
+      int ret = madvise(nextPage, end - nextPage, MADV_PAGEOUT);
+      if (ret == 0)
+         return true;
+      if (trace)
+         TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for shared class cache; errno: %d", errno);
+      // Temporary failure, don't disable disclaim permanently if this happens.
+      if (errno == EAGAIN)
+         return true;
+      }
+   return false;
+   }
+
+int32_t TR_J9SharedCache::disclaimSharedCaches()
+   {
+   int32_t numDisclaimed = 0;
+
+   if (!_disclaimEnabled)
+      return numDisclaimed;
+
+   J9SharedClassCacheDescriptor *scHead = getCacheDescriptorList();
+   J9SharedClassCacheDescriptor *scCur = scHead;
+   PORT_ACCESS_FROM_JAVAVM(_javaVM); // for j9vmem_supported_page_sizes
+   UDATA pageSize = j9vmem_supported_page_sizes()[0];
+   bool trace = TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance);
+
+   do
+      {
+      uint8_t *rwStart = RWUPDATEPTR(scCur->cacheStartAddress);
+      uint8_t *rwEnd = SEGUPDATEPTR(scCur->cacheStartAddress);
+      if (!disclaim(rwStart, rwEnd, pageSize, trace))
+         {
+         if (trace)
+            TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling shared class cache disclaiming from now on");
+         _disclaimEnabled = false;
+         break;
+         }
+      numDisclaimed++;
+      uint8_t *updateStart = UPDATEPTR(scCur->cacheStartAddress);
+      uint8_t *updateEnd = CAEND(scCur->cacheStartAddress);
+      if (!disclaim(updateStart, updateEnd, pageSize, trace))
+         {
+         if (trace)
+            TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling shared class cache disclaiming from now on");
+         _disclaimEnabled = false;
+         break;
+         }
+      numDisclaimed++;
+      scCur = scCur->next;
+      }
+   while (scCur != scHead);
+
+   return numDisclaimed;
+   }
+#endif // defined(LINUX)
+
 TR_YesNoMaybe TR_J9SharedCache::isSharedCacheDisabledBecauseFull(TR::CompilationInfo *compInfo)
    {
    if (_sharedCacheDisabledBecauseFull == TR_maybe)
@@ -186,6 +260,9 @@ TR_J9SharedCache::TR_J9SharedCache(TR_J9VMBase *fe)
    _aotStats = fe->getPrivateConfig()->aotStats;
    _sharedCacheConfig = _javaVM->sharedClassConfig;
    _numDigitsForCacheOffsets = 8;
+#if defined(LINUX)
+   _disclaimEnabled = TR::Options::getCmdLineOptions()->getOption(TR_EnableSharedCacheDisclaiming);
+#endif
 
 #if defined(J9VM_OPT_JITSERVER)
    TR_ASSERT_FATAL(_sharedCacheConfig || _compInfo->getPersistentInfo()->getRemoteCompilationMode() == JITServer::SERVER

runtime/compiler/env/J9SharedCache.hpp

@@ -105,6 +105,34 @@ class TR_J9SharedCache : public TR_SharedCache
 
    static void validateAOTHeader(J9JITConfig *jitConfig, J9VMThread *vmThread, TR::CompilationInfo *compInfo);
 
+#if defined(LINUX)
+   /**
+    * \brief Returns whether SCC disclaiming is enabled.
+    *
+    * This function returns a boolean that represents whether SCC disclaiming is enabled or not.
+    * SCC disclaiming may be enabled or disabled via an option, and if enabled, may later be
+    * disabled due to an error.
+    *
+    * \return True if SCC disclaiming is enabled, false otherwise.
+    */
+   virtual bool isDisclaimEnabled() { return _disclaimEnabled; }
+
+   /**
+    * \brief Disclaims SCC memory pages, causing them to be paged out of the resident set.
+    *
+    * This function causes the occupied parts of the SCC to be disclaimed. For each layer
+    * the ROM class area which starts at the beginning of the layer and extends forward,
+    * and the metadata area which starts at the end of the layer and extends backwards,
+    * are both disclaimed, causing the underlying memory pages to be paged out of the
+    * resident set.
+    *
+    * This function does nothing if SCC disclaiming is disabled.
+    *
+    * \return The number of areas disclaimed.
+    */
+   virtual int32_t disclaimSharedCaches();
+#endif
+
    /**
     * \brief Converts a shared cache offset, calculated from the end of the SCC, into the
     *        metadata section of the SCC into a pointer.
@@ -421,6 +449,8 @@ class TR_J9SharedCache : public TR_SharedCache
    virtual J9SharedClassCacheDescriptor *getCacheDescriptorList();
 
 protected:
+   static bool disclaim(const uint8_t *start, const uint8_t *end, UDATA pageSize, bool trace);
+
    /**
     * \brief Helper method; used to check if a pointer is within the SCC
     *
@@ -620,6 +650,10 @@ class TR_J9SharedCache : public TR_SharedCache
    uint32_t _logLevel;
    bool _verboseHints;
 
+#if defined(LINUX)
+   bool _disclaimEnabled;
+#endif
+
    static TR_J9SharedCacheDisabledReason _sharedCacheState;
    static TR_YesNoMaybe                  _sharedCacheDisabledBecauseFull;
    static UDATA                          _storeSharedDataFailedLength;

runtime/compiler/runtime/J9CodeCache.cpp

@@ -918,7 +918,7 @@ J9::CodeCache::disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap)
       if (trace)
          TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for code cache");
 
-      if (ret == EINVAL)
+      if (errno != EAGAIN)
          {
          manager->setDisclaimEnabled(false); // Don't try to disclaim again, since support seems to be missing
          if (trace)