Retained size by ClassLoaders

CMakeLists.txt

@@ -17,6 +17,8 @@ endif ()
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}")
 find_package(JNI11)
 include_directories(${JNI_INCLUDE_DIRS})
+include_directories("/usr/lib/jvm/java-1.11.0-openjdk-amd64/include")
+include_directories("/usr/lib/jvm/java-1.11.0-openjdk-amd64/include/linux")
 
 add_library(memory_agent SHARED
         src/agent.cpp
@@ -36,6 +38,8 @@ add_library(memory_agent SHARED
         src/sizes/tag_info_array.cpp
         src/sizes/retained_size_by_classes.h
         src/sizes/retained_size_by_classes.cpp
+        src/sizes/retained_size_by_classloaders.h
+        src/sizes/retained_size_by_classloaders.cpp
         src/sizes/retained_size_action.h
         src/sizes/retained_size_by_objects.h
         src/sizes/retained_size_by_objects.cpp

src/agent.cpp

@@ -14,6 +14,7 @@
 #include "sizes/retained_size_and_held_objects.h"
 #include "sizes/retained_size_by_classes.h"
 #include "sizes/retained_size_by_objects.h"
+#include "sizes/retained_size_by_classloaders.h"
 #include "allocation_sampling.h"
 
 #pragma ide diagnostic ignored "OCUnusedGlobalDeclarationInspection"
@@ -140,6 +141,13 @@ JNIEXPORT jboolean JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentProxy_c
     return (jboolean) 1;
 }
 
+extern "C"
+JNIEXPORT jboolean JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentProxy_canGetRetainedSizeByClassLoaders(
+        JNIEnv *env,
+        jobject thisObject) {
+    return (jboolean) 1;
+}
+
 extern "C"
 JNIEXPORT jboolean JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentProxy_canGetShallowSizeByClasses(
         JNIEnv *env,
@@ -196,6 +204,14 @@ JNIEXPORT jobjectArray JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentPro
     return RetainedSizeByClassesAction(env, gdata->jvmti, thisObject).run(classesArray);
 }
 
+extern "C"
+JNIEXPORT jobjectArray JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentProxy_getRetainedSizeByClassLoaders(
+        JNIEnv *env,
+        jobject thisObject,
+        jobjectArray classLoadersArray) {
+    return RetainedSizeByClassLoadersAction(env, gdata->jvmti, thisObject).run(classLoadersArray);
+}
+
 extern "C"
 JNIEXPORT jobjectArray JNICALL Java_com_intellij_memory_agent_IdeaNativeAgentProxy_getShallowAndRetainedSizeByClasses(
         JNIEnv *env,

src/sizes/retained_size_action.cpp

@@ -98,6 +98,10 @@ jint JNICALL clearTag(jlong classTag, jlong size, jlong *tagPtr, jint length, vo
     if (*tagPtr == 0) {
         return JVMTI_ITERATION_CONTINUE;
     }
+//    if (*tagPtr == 13) {
+//        *tagPtr = 0;
+//        return JVMTI_ITERATION_CONTINUE;
+//    }
 
     tagToPointer(*tagPtr)->unref();
     *tagPtr = 0;
@@ -120,7 +124,13 @@ jint JNICALL tagObjectOfTaggedClass(jlong classTag, jlong size, jlong *tagPtr, j
     if (pClassTag && *tagPtr == 0) {
         *tagPtr = pointerToTag(pClassTag->createStartTag());
     }
+    return JVMTI_ITERATION_CONTINUE;
+}
 
+jint JNICALL tagObjectOfTaggedClassSimple(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData) {
+    if (classTag == 13) {
+        *tagPtr = 13;
+    }
     return JVMTI_ITERATION_CONTINUE;
 }
 

src/sizes/retained_size_action.h

@@ -28,6 +28,7 @@ jint JNICALL clearTag               (jlong classTag, jlong size, jlong *tagPtr,
 jint JNICALL retagStartObjects      (jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
 
 jint JNICALL tagObjectOfTaggedClass (jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
+jint JNICALL tagObjectOfTaggedClassSimple (jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
 
 jvmtiError walkHeapFromObjects      (jvmtiEnv *jvmti, const std::vector<jobject> &objects, const CancellationChecker &cancellationChecker);
 
@@ -72,6 +73,35 @@ class RetainedSizeAction : public MemoryAgentAction<RESULT_TYPE, jobjectArray> {
         return JVMTI_ERROR_NONE;
     }
 
+    jvmtiError createTagsForClassLoadersClasses(JNIEnv *env, jvmtiEnv *jvmti, jobjectArray classesArray, jsize classLoaderIndex) {
+        for (jsize i = 0; i < this->env->GetArrayLength(classesArray); i++) {
+            jobject classObject = this->env->GetObjectArrayElement(classesArray, i);
+            jvmtiError err = tagClass(env, jvmti, classObject, [classLoaderIndex](jlong oldTag) -> jlong {
+                ClassTag *classTag = tagToClassTagPointer(oldTag);
+                if (classTag != nullptr) {
+                    classTag->ids.push_back(classLoaderIndex);
+                } else {
+                    return pointerToTag(ClassTag::create(static_cast<query_size_t>(classLoaderIndex)));
+                }
+
+                return 0;
+            });
+            if (err != JVMTI_ERROR_NONE) return err;
+        }
+        return JVMTI_ERROR_NONE;
+    }
+
+    jvmtiError createTagsForClassLoadersClassesSimple(JNIEnv *env, jvmtiEnv *jvmti, jobjectArray classesArray) {
+        for (jsize i = 0; i < this->env->GetArrayLength(classesArray); i++) {
+            jobject classObject = this->env->GetObjectArrayElement(classesArray, i);
+            jvmtiError err = tagClass(env, jvmti, classObject, [](jlong oldTag) -> jlong {
+                return 13;
+            });
+            if (err != JVMTI_ERROR_NONE) return err;
+        }
+        return JVMTI_ERROR_NONE;
+    }
+
     jvmtiError tagObjectsOfClasses(jobjectArray classesArray) {
         debug("tag objects of classes");
         jvmtiError err = createTagsForClasses(this->env, this->jvmti, classesArray);
@@ -80,6 +110,22 @@ class RetainedSizeAction : public MemoryAgentAction<RESULT_TYPE, jobjectArray> {
         return this->IterateThroughHeap(0, nullptr, tagObjectOfTaggedClass, nullptr);
     }
 
+    jvmtiError tagObjectsOfClassLoaderClasses(jobjectArray classesArray, jsize classLoaderIndex) {
+        debug("tag objects of classes");
+        jvmtiError err = createTagsForClassLoadersClasses(this->env, this->jvmti, classesArray, classLoaderIndex);
+        if (err != JVMTI_ERROR_NONE) return err;
+
+        return this->IterateThroughHeap(0, nullptr, tagObjectOfTaggedClass, nullptr);
+    }
+
+    jvmtiError tagObjectsOfClassLoaderClassesSimple(jobjectArray classesArray) {
+        debug("tag objects of classes");
+        jvmtiError err = createTagsForClassLoadersClassesSimple(this->env, this->jvmti, classesArray);
+        if (err != JVMTI_ERROR_NONE) return err;
+
+        return this->IterateThroughHeap(0, nullptr, tagObjectOfTaggedClassSimple, nullptr);
+    }
+
     jvmtiError tagHeap() {
         jvmtiError err = this->FollowReferences(0, nullptr, nullptr, getTagsWithNewInfo, nullptr, "find objects with new info");
         if (err != JVMTI_ERROR_NONE) return err;

src/sizes/retained_size_by_classes.cpp

@@ -21,6 +21,11 @@ jint JNICALL visitObject(jlong classTag, jlong size, jlong *tagPtr, jint length,
     return JVMTI_ITERATION_CONTINUE;
 }
 
+jint JNICALL countHeapSize(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData) {
+    *reinterpret_cast<jlong*>(userData) += size;
+    return JVMTI_ITERATION_CONTINUE;
+}
+
 
 jint JNICALL visitObjectForShallowAndRetainedSize(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData) {
     if (*tagPtr == 0) {

src/sizes/retained_size_by_classes.h

@@ -26,5 +26,6 @@ class RetainedAndShallowSizeByClassesAction : public RetainedSizeAction<jobjectA
 };
 
 jint JNICALL visitObject(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
+jint JNICALL countHeapSize(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
 
 #endif //MEMORY_AGENT_RETAINED_SIZE_BY_CLASSES_H

src/sizes/retained_size_by_classloaders.cpp

@@ -0,0 +1,158 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+
+#include <vector>
+#include "retained_size_by_classloaders.h"
+#include <algorithm>
+#include <numeric>
+
+
+jvmtiError RetainedSizeByClassLoadersAction::tagObjectsByClassLoader(jclass *classes, jint* cnt, jobject classLoader, jsize classLoaderIndex) {
+    jvmtiError err = JVMTI_ERROR_NONE;
+    std::vector<jclass> filteredClasses;
+    for (jsize i = 0; i < *cnt; i++) {
+        jobject rootClassLoader = getClassLoader(env, classes[i]);
+        if (!env->IsSameObject(rootClassLoader, NULL)) {
+            if (isEqual(env, classLoader, rootClassLoader)) {
+                filteredClasses.emplace_back(classes[i]);
+            }
+        }
+    }
+
+    err = tagObjectsOfClassLoaderClasses(toJavaArray(env, filteredClasses), classLoaderIndex);
+    if (!isOk(err)) return err;
+    if (shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+    return err;
+}
+
+jvmtiError RetainedSizeByClassLoadersAction::tagObjectsByClassLoaderSimple(jclass *classes, jint* cnt, jobject classLoader) {
+    jvmtiError err = JVMTI_ERROR_NONE;
+    std::vector<jclass> filteredClasses;
+    for (jsize i = 0; i < *cnt; i++) {
+        jobject rootClassLoader = getClassLoader(env, classes[i]);
+        if (!env->IsSameObject(rootClassLoader, NULL)) {
+            if (isEqual(env, classLoader, rootClassLoader)) {
+                filteredClasses.emplace_back(classes[i]);
+            }
+        }
+    }
+
+    debug("tag objects of classes");
+    err = createTagsForClassLoadersClassesSimple(this->env, this->jvmti, toJavaArray(env, filteredClasses));
+    if (err != JVMTI_ERROR_NONE) return err;
+
+    err = this->IterateThroughHeap(0, nullptr, tagObjectOfTaggedClassSimple, nullptr);
+    if (!isOk(err)) return err;
+    if (shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+    return err;
+}
+
+jlong RetainedSizeByClassLoadersAction::tagOtherObjects(jclass *classes, jint* cnt, jobjectArray classLoadersArray) {
+    jvmtiError err = JVMTI_ERROR_NONE;
+    std::vector<jclass> filteredClasses;
+    for (jsize i = 0; i < *cnt; i++) {
+        jobject rootClassLoader = getClassLoader(env, classes[i]);
+        if (!env->IsSameObject(rootClassLoader, NULL)) {
+            for (jsize j = 0; j < env->GetArrayLength(classLoadersArray); j++) {
+                if (isEqual(env, env->GetObjectArrayElement(classLoadersArray, j), rootClassLoader)) {
+                    filteredClasses.emplace_back(classes[i]);
+                    break;
+                }
+            }
+        }
+    }
+
+    std::vector<jclass> nonfilteredClasses;
+    for (jsize i = 0; i < *cnt; i++) {
+        if (std::find(filteredClasses.begin(), filteredClasses.end(), classes[i]) == filteredClasses.end()) {
+            nonfilteredClasses.emplace_back(classes[i]);
+        }
+    }
+    std::cout << "SIZE: " << nonfilteredClasses.size() << std::endl;
+
+    err = tagObjectsOfClassLoaderClasses(toJavaArray(env, nonfilteredClasses), env->GetArrayLength(classLoadersArray));
+}
+
+jlong RetainedSizeByClassLoadersAction::getHeapSize(){
+    jlong totalSize = 0;
+    IterateThroughHeap(0, nullptr, countHeapSize, &totalSize,
+                       "calculate heap size");
+    return totalSize;
+}
+
+RetainedSizeByClassLoadersAction::RetainedSizeByClassLoadersAction(JNIEnv *env, jvmtiEnv *jvmti, jobject object)
+        : RetainedSizeAction(env, jvmti, object), retainedSizeByObjectsMergedAction(env, jvmti, object), retainedSizeByObjectsAction(env, jvmti, object) {
+
+}
+
+jvmtiError RetainedSizeByClassLoadersAction::getRetainedSizeByClassLoaders(jobjectArray classLoadersArray,
+                                                                           std::vector <jlong> &result) {
+    jvmtiError err = JVMTI_ERROR_NONE;
+    jclass *classes;
+    jint cnt;
+    err = jvmti->GetLoadedClasses(&cnt, &classes);
+    for (jsize i = 0; i < env->GetArrayLength(classLoadersArray); i++) {
+        jvmtiError err = tagObjectsByClassLoader(classes, &cnt, env->GetObjectArrayElement(classLoadersArray, i), i);
+    }
+    tagOtherObjects(classes, &cnt, classLoadersArray);
+
+    if (!isOk(err)) return err;
+    if (shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+
+    err = tagHeap();
+    if (!isOk(err)) return err;
+    if (shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+
+    result.resize(env->GetArrayLength(classLoadersArray) + 1);
+    err = IterateThroughHeap(JVMTI_HEAP_FILTER_UNTAGGED, nullptr, visitObject, result.data(),
+                             "calculate retained size");
+    return err;
+}
+
+jvmtiError RetainedSizeByClassLoadersAction::getRetainedSizeByClassLoaderSimple(jobject classLoader,
+                                                                           jlong* result) {
+    jvmtiError err = JVMTI_ERROR_NONE;
+    jclass *classes;
+    jint cnt;
+    err = jvmti->GetLoadedClasses(&cnt, &classes);
+    err = tagObjectsByClassLoaderSimple(classes, &cnt, classLoader);
+    std::cout << "tagged" << std::endl;
+
+    if (!isOk(err)) return err;
+    if (shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+
+    std::vector<jobject> objects;
+   err = getObjectsByTags(this->jvmti, std::vector<jlong>{13}, objects);
+    std::cout << "get objects" << std::endl;
+    if (err != JVMTI_ERROR_NONE) return err;
+    if (this->shouldStopExecution()) return MEMORY_AGENT_INTERRUPTED_ERROR;
+    removeAllTagsFromHeap(jvmti, nullptr);
+
+    std::cout << "remove tags" << std::endl;
+    std::cout << objects.size() << std::endl;
+    auto r = retainedSizeByObjectsMergedAction.run(toJavaArray(env, objects));
+    result = reinterpret_cast<jlong*>(env->GetObjectArrayElement(r, 0));
+    std::cout << result << " " << *result << std::endl;
+    removeAllTagsFromHeap(jvmti, nullptr);
+    return err;
+}
+
+jlongArray RetainedSizeByClassLoadersAction::executeOperation(jobjectArray classLoadersArray) {
+    std::vector <jlong> result;
+    jvmtiError err = getRetainedSizeByClassLoaders(classLoadersArray, result);
+    std::cout << "CLASSLOADERS CLASSES: " << result[0] <<std::endl;
+    std::vector <jlong> otherRes;
+    for (jsize i = 0; i < env->GetArrayLength(classLoadersArray); i++) {
+        jlong res;
+        jvmtiError err = getRetainedSizeByClassLoaderSimple(env->GetObjectArrayElement(classLoadersArray, i), &res);
+        otherRes.push_back(res);
+    }
+    if (!isOk(err)) {
+        handleError(jvmti, err, "Could not estimate retained size by classLoaders");
+        return env->NewLongArray(0);
+    }
+    //std::cout << "HEAP SIZE: " << getHeapSize() << std::endl;
+    std::cout << "CLASSLOADERS CLASSES CHECK: " << std::accumulate(otherRes.begin(), otherRes.end(), 0l) <<std::endl;
+    std::cout << "OTHER CLASSES: " << result.back() <<std::endl;
+    result.pop_back();
+    return toJavaArray(env, result);
+}

src/sizes/retained_size_by_classloaders.h

@@ -0,0 +1,33 @@
+// Copyright 2000-2018 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+
+#ifndef MEMORY_AGENT_RETAINED_SIZE_BY_CLASSLOADERS_H
+#define MEMORY_AGENT_RETAINED_SIZE_BY_CLASSLOADERS_H
+
+#include "../memory_agent_action.h"
+#include "retained_size_action.h"
+#include "retained_size_by_objects_merged.h"
+#include "retained_size_by_objects.h"
+#include "retained_size_by_classes.h"
+
+
+class RetainedSizeByClassLoadersAction : public RetainedSizeAction<jlongArray> {
+public:
+    RetainedSizeByClassLoadersAction(JNIEnv *env, jvmtiEnv *jvmti, jobject object);
+    RetainedSizeByObjectsMergedAction retainedSizeByObjectsMergedAction;
+    RetainedSizeByObjectsAction retainedSizeByObjectsAction;
+
+private:
+    jlongArray executeOperation(jobjectArray classLoadersArray) override;
+    jvmtiError getRetainedSizeByClassLoaders(jobjectArray classLoadersArray, std::vector<jlong> &result);
+    jvmtiError getRetainedSizeByClassLoaderSimple(jobject classLoader, jlong* result);
+    jvmtiError tagObjectsByClassLoader(jclass *classes, jint* cnt, jobject classLoader, jsize classLoaderIndex);
+    jvmtiError tagObjectsByClassLoaderSimple(jclass *classes, jint* cnt, jobject classLoader);
+    jlong getHeapSize();
+    jlong tagOtherObjects(jclass *classes, jint* cnt, jobjectArray classLoadersArray);
+};
+
+jint JNICALL visitObject(jlong classTag, jlong size, jlong *tagPtr, jint length, void *userData);
+
+
+
+#endif //MEMORY_AGENT_RETAINED_SIZE_BY_CLASSLOADERS_H