diff --git a/runtime/compiler/control/HookedByTheJit.cpp b/runtime/compiler/control/HookedByTheJit.cpp index ad8a64bc9bb..1c585bd2258 100644 --- a/runtime/compiler/control/HookedByTheJit.cpp +++ b/runtime/compiler/control/HookedByTheJit.cpp @@ -4515,14 +4515,26 @@ size_t getRSS_Kb() return rss; } +#if defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX) +void disclaimSharedClassCache(TR_J9SharedCache *sharedCache, uint64_t crtElapsedTime) + { + size_t rssBefore = getRSS_Kb(); + int32_t numDisclaimed = sharedCache->disclaimSharedCaches(); + size_t rssAfter = getRSS_Kb(); + if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance)) + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d SCC segments RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%", + (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore)); + } +#endif // defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX) + void disclaimDataCaches(uint64_t crtElapsedTime) { size_t rssBefore = getRSS_Kb(); int numDisclaimed = TR_DataCacheManager::getManager()->disclaimAllDataCaches(); size_t rssAfter = getRSS_Kb(); if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance)) - TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Data Cache segments RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB", - (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter); + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Data Cache segments RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%", + (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore)); } void disclaimIProfilerSegments(uint64_t crtElapsedTime) @@ -4535,8 +4547,8 @@ void disclaimIProfilerSegments(uint64_t crtElapsedTime) int numSegDisclaimed = iprofilerAllocator->disclaimAllSegments(); size_t rssAfter = getRSS_Kb(); if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance)) - TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d IProfiler segments out of %d. RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB", - (uint32_t)crtElapsedTime, numSegDisclaimed, iprofilerAllocator->getNumSegments(), rssBefore, rssAfter, rssBefore - rssAfter); + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d IProfiler segments out of %d. RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%", + (uint32_t)crtElapsedTime, numSegDisclaimed, iprofilerAllocator->getNumSegments(), rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore)); } } @@ -4546,7 +4558,7 @@ void disclaimCodeCaches(uint64_t crtElapsedTime) int numDisclaimed = TR::CodeCacheManager::instance()->disclaimAllCodeCaches(); size_t rssAfter = getRSS_Kb(); if (TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance)) - TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zu KB = %5.2f%%", + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "t=%u JIT disclaimed %d Code Caches RSS before=%zu KB, RSS after=%zu KB, delta=%zd KB = %5.2f%%", (uint32_t)crtElapsedTime, numDisclaimed, rssBefore, rssAfter, rssBefore - rssAfter, ((long)(rssAfter - rssBefore) * 100.0 / rssBefore)); } @@ -4557,6 +4569,7 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime, static uint64_t lastCodeCacheDisclaimTime = 0; static int32_t lastNumAllocatedCodeCaches = 0; static uint64_t lastIProfilerDisclaimTime = 0; + static uint64_t lastSCCDisclaimTime = 0; static uint32_t lastNumCompilationsDuringIProfilerDisclaim = 0; J9JITConfig *jitConfig = compInfo->getJITConfig(); @@ -4571,6 +4584,20 @@ void memoryDisclaimLogic(TR::CompilationInfo *compInfo, uint64_t crtElapsedTime, if (javaVM->phase != J9VM_PHASE_NOT_STARTUP || jitState == STARTUP_STATE) return; +#if defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX) + TR_J9VMBase *fej9 = TR_J9VMBase::get(jitConfig, compInfo->getSamplerThread(), TR_J9VMBase::AOT_VM); + TR_J9SharedCache *sharedCache = fej9->sharedCache(); + if (sharedCache && sharedCache->isDisclaimEnabled()) + { + // Disclaim if there was a large time interval since the last disclaim + if (crtElapsedTime > lastSCCDisclaimTime + 12 * TR::Options::_minTimeBetweenMemoryDisclaims) + { + disclaimSharedClassCache(sharedCache, crtElapsedTime); + lastSCCDisclaimTime = crtElapsedTime; + } + } +#endif // defined(J9VM_OPT_SHARED_CLASSES) && defined(LINUX) + if (TR_DataCacheManager::getManager()->isDisclaimEnabled()) { // Ensure we don't do it too often diff --git a/runtime/compiler/control/J9Options.cpp b/runtime/compiler/control/J9Options.cpp index 7799438d6b4..5f5e21d25aa 100644 --- a/runtime/compiler/control/J9Options.cpp +++ b/runtime/compiler/control/J9Options.cpp @@ -2929,7 +2929,8 @@ J9::Options::fePostProcessJIT(void * base) if (!self()->getOption(TR_DisableDataCacheDisclaiming) || !self()->getOption(TR_DisableIProfilerDataDisclaiming) || - self()->getOption(TR_EnableCodeCacheDisclaiming)) + self()->getOption(TR_EnableCodeCacheDisclaiming) || + self()->getOption(TR_EnableSharedCacheDisclaiming)) { // Check requirements for memory disclaiming (Linux kernel and default page size) TR::Options::disableMemoryDisclaimIfNeeded(jitConfig); @@ -3002,6 +3003,7 @@ J9::Options::disableMemoryDisclaimIfNeeded(J9JITConfig *jitConfig) TR::Options::getCmdLineOptions()->setOption(TR_DisableDataCacheDisclaiming); TR::Options::getCmdLineOptions()->setOption(TR_DisableIProfilerDataDisclaiming); TR::Options::getCmdLineOptions()->setOption(TR_EnableCodeCacheDisclaiming, false); + TR::Options::getCmdLineOptions()->setOption(TR_EnableSharedCacheDisclaiming, false); } return shouldDisableMemoryDisclaim; } diff --git a/runtime/compiler/control/OptionsPostRestore.cpp b/runtime/compiler/control/OptionsPostRestore.cpp index f4bdf93d89f..6a02796062b 100644 --- a/runtime/compiler/control/OptionsPostRestore.cpp +++ b/runtime/compiler/control/OptionsPostRestore.cpp @@ -830,7 +830,8 @@ J9::OptionsPostRestore::postProcessInternalCompilerOptions() if (!TR::Options::getCmdLineOptions()->getOption(TR_DisableDataCacheDisclaiming) || !TR::Options::getCmdLineOptions()->getOption(TR_DisableIProfilerDataDisclaiming) || - TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming)) + TR::Options::getCmdLineOptions()->getOption(TR_EnableCodeCacheDisclaiming) || + TR::Options::getCmdLineOptions()->getOption(TR_EnableSharedCacheDisclaiming)) { TR::Options::disableMemoryDisclaimIfNeeded(_jitConfig); } diff --git a/runtime/compiler/env/J9SharedCache.cpp b/runtime/compiler/env/J9SharedCache.cpp index a63b912f2d5..e8ec4dbb9f3 100644 --- a/runtime/compiler/env/J9SharedCache.cpp +++ b/runtime/compiler/env/J9SharedCache.cpp @@ -48,6 +48,17 @@ #include "runtime/JITServerAOTDeserializer.hpp" #endif +// for madvise +#ifdef LINUX +#include +#ifndef MADV_NOHUGEPAGE +#define MADV_NOHUGEPAGE 15 +#endif // MADV_NOHUGEPAGE +#ifndef MADV_PAGEOUT +#define MADV_PAGEOUT 21 +#endif // MADV_PAGEOUT +#endif + #define LOG(logLevel, format, ...) \ if (_logLevel >= logLevel) \ { \ @@ -55,6 +66,8 @@ } // From CompositeCache.cpp +#define RWUPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->readWriteSRP) +#define CAEND(ca) (((uint8_t *)(ca)) + (ca)->totalBytes) #define UPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->updateSRP) #define SEGUPDATEPTR(ca) (((uint8_t *)(ca)) + (ca)->segmentSRP) @@ -105,6 +118,67 @@ TR_J9SharedCache::validateAOTHeader(J9JITConfig *jitConfig, J9VMThread *vmThread } } +#if defined(LINUX) +bool TR_J9SharedCache::disclaim(const uint8_t *start, const uint8_t *end, UDATA pageSize, bool trace) + { + uint8_t *nextPage = (uint8_t *)(((UDATA)start + (pageSize - 1)) & ~(pageSize - 1)); + if (nextPage < end) + { + int ret = madvise(nextPage, end - nextPage, MADV_PAGEOUT); + if (ret == 0) + return true; + if (trace) + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for shared class cache; errno: %d", errno); + // Temporary failure, don't disable disclaim permanently if this happens. + if (errno == EAGAIN) + return true; + } + return false; + } + +int32_t TR_J9SharedCache::disclaimSharedCaches() + { + int32_t numDisclaimed = 0; + + if (!_disclaimEnabled) + return numDisclaimed; + + J9SharedClassCacheDescriptor *scHead = getCacheDescriptorList(); + J9SharedClassCacheDescriptor *scCur = scHead; + PORT_ACCESS_FROM_JAVAVM(_javaVM); // for j9vmem_supported_page_sizes + UDATA pageSize = j9vmem_supported_page_sizes()[0]; + bool trace = TR::Options::getCmdLineOptions()->getVerboseOption(TR_VerbosePerformance); + + do + { + uint8_t *rwStart = RWUPDATEPTR(scCur->cacheStartAddress); + uint8_t *rwEnd = SEGUPDATEPTR(scCur->cacheStartAddress); + if (!disclaim(rwStart, rwEnd, pageSize, trace)) + { + if (trace) + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling shared class cache disclaiming from now on"); + _disclaimEnabled = false; + break; + } + numDisclaimed++; + uint8_t *updateStart = UPDATEPTR(scCur->cacheStartAddress); + uint8_t *updateEnd = CAEND(scCur->cacheStartAddress); + if (!disclaim(updateStart, updateEnd, pageSize, trace)) + { + if (trace) + TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Disabling shared class cache disclaiming from now on"); + _disclaimEnabled = false; + break; + } + numDisclaimed++; + scCur = scCur->next; + } + while (scCur != scHead); + + return numDisclaimed; + } +#endif // defined(LINUX) + TR_YesNoMaybe TR_J9SharedCache::isSharedCacheDisabledBecauseFull(TR::CompilationInfo *compInfo) { if (_sharedCacheDisabledBecauseFull == TR_maybe) @@ -186,6 +260,9 @@ TR_J9SharedCache::TR_J9SharedCache(TR_J9VMBase *fe) _aotStats = fe->getPrivateConfig()->aotStats; _sharedCacheConfig = _javaVM->sharedClassConfig; _numDigitsForCacheOffsets = 8; +#if defined(LINUX) + _disclaimEnabled = TR::Options::getCmdLineOptions()->getOption(TR_EnableSharedCacheDisclaiming); +#endif #if defined(J9VM_OPT_JITSERVER) TR_ASSERT_FATAL(_sharedCacheConfig || _compInfo->getPersistentInfo()->getRemoteCompilationMode() == JITServer::SERVER diff --git a/runtime/compiler/env/J9SharedCache.hpp b/runtime/compiler/env/J9SharedCache.hpp index 370a19cce12..3b6a37cd4e7 100644 --- a/runtime/compiler/env/J9SharedCache.hpp +++ b/runtime/compiler/env/J9SharedCache.hpp @@ -105,6 +105,34 @@ class TR_J9SharedCache : public TR_SharedCache static void validateAOTHeader(J9JITConfig *jitConfig, J9VMThread *vmThread, TR::CompilationInfo *compInfo); +#if defined(LINUX) + /** + * \brief Returns whether SCC disclaiming is enabled. + * + * This function returns a boolean that represents whether SCC disclaiming is enabled or not. + * SCC disclaiming may be enabled or disabled via an option, and if enabled, may later be + * disabled due to an error. + * + * \return True if SCC disclaiming is enabled, false otherwise. + */ + virtual bool isDisclaimEnabled() { return _disclaimEnabled; } + + /** + * \brief Disclaims SCC memory pages, causing them to be paged out of the resident set. + * + * This function causes the occupied parts of the SCC to be disclaimed. For each layer + * the ROM class area which starts at the beginning of the layer and extends forward, + * and the metadata area which starts at the end of the layer and extends backwards, + * are both disclaimed, causing the underlying memory pages to be paged out of the + * resident set. + * + * This function does nothing if SCC disclaiming is disabled. + * + * \return The number of areas disclaimed. + */ + virtual int32_t disclaimSharedCaches(); +#endif + /** * \brief Converts a shared cache offset, calculated from the end of the SCC, into the * metadata section of the SCC into a pointer. @@ -421,6 +449,8 @@ class TR_J9SharedCache : public TR_SharedCache virtual J9SharedClassCacheDescriptor *getCacheDescriptorList(); protected: + static bool disclaim(const uint8_t *start, const uint8_t *end, UDATA pageSize, bool trace); + /** * \brief Helper method; used to check if a pointer is within the SCC * @@ -620,6 +650,10 @@ class TR_J9SharedCache : public TR_SharedCache uint32_t _logLevel; bool _verboseHints; +#if defined(LINUX) + bool _disclaimEnabled; +#endif + static TR_J9SharedCacheDisabledReason _sharedCacheState; static TR_YesNoMaybe _sharedCacheDisabledBecauseFull; static UDATA _storeSharedDataFailedLength; diff --git a/runtime/compiler/runtime/J9CodeCache.cpp b/runtime/compiler/runtime/J9CodeCache.cpp index b66df9acca1..6cf07feb679 100644 --- a/runtime/compiler/runtime/J9CodeCache.cpp +++ b/runtime/compiler/runtime/J9CodeCache.cpp @@ -918,7 +918,7 @@ J9::CodeCache::disclaim(TR::CodeCacheManager *manager, bool canDisclaimOnSwap) if (trace) TR_VerboseLog::writeLineLocked(TR_Vlog_PERF, "WARNING: Failed to use madvise to disclaim memory for code cache"); - if (ret == EINVAL) + if (errno != EAGAIN) { manager->setDisclaimEnabled(false); // Don't try to disclaim again, since support seems to be missing if (trace)