diff --git a/runtime/gc/gctable.c b/runtime/gc/gctable.c index c42aa2b1d0c..f81308ae981 100644 --- a/runtime/gc/gctable.c +++ b/runtime/gc/gctable.c @@ -143,6 +143,7 @@ J9MemoryManagerFunctions MemoryManagerFunctions = { #endif /* !J9VM_ENV_DATA64 */ j9gc_objaccess_indexableStoreObject, j9gc_objaccess_indexableStoreAddress, + j9gc_objaccess_indexableDataDisplacement, j9gc_objaccess_mixedObjectReadI32, j9gc_objaccess_mixedObjectReadU32, j9gc_objaccess_mixedObjectReadI64, diff --git a/runtime/gc_base/ObjectAccessBarrier.hpp b/runtime/gc_base/ObjectAccessBarrier.hpp index e79ba09f46f..e4ceaf62848 100644 --- a/runtime/gc_base/ObjectAccessBarrier.hpp +++ b/runtime/gc_base/ObjectAccessBarrier.hpp @@ -226,6 +226,10 @@ class MM_ObjectAccessBarrier : public MM_BaseVirtual virtual void staticStoreU64(J9VMThread *vmThread, J9Class *clazz, U_64 *destSlot, U_64 value, bool isVolatile=false); virtual void staticStoreI64(J9VMThread *vmThread, J9Class *clazz, I_64 *destSlot, I_64 value, bool isVolatile=false); + MMINLINE virtual IDATA indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) + { + return (IDATA)(((UDATA)dst) - ((UDATA)src)); + } virtual U_8 *getArrayObjectDataAddress(J9VMThread *vmThread, J9IndexableObject *arrayObject); virtual j9objectmonitor_t *getLockwordAddress(J9VMThread *vmThread, J9Object *object); virtual void cloneObject(J9VMThread *vmThread, J9Object *srcObject, J9Object *destObject); diff --git a/runtime/gc_base/accessBarrier.cpp b/runtime/gc_base/accessBarrier.cpp index b51463ca7a2..870f201dcaa 100644 --- a/runtime/gc_base/accessBarrier.cpp +++ b/runtime/gc_base/accessBarrier.cpp @@ -341,6 +341,24 @@ j9gc_objaccess_staticStoreU64Split(J9VMThread *vmThread, J9Class *clazz, U_64 *d barrier->staticStoreU64(vmThread, clazz, destSlot, value64, 0 != isVolatile); } +/** + * Returns the displacement for the data of moved array object. + * Used by the JIT, should only be called for off heap enabled cases, + * For adjacent Array, displacement = dst - src + * For Off-heap Array, displacement = 0. + * + * @param vmThread Pointer to the current J9VMThread + * @param src Pointer to the array object before moving + * @param dst Pointer to the array object after moving + * @return displacement + */ +IDATA +j9gc_objaccess_indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) +{ + MM_ObjectAccessBarrier *barrier = MM_GCExtensions::getExtensions(vmThread)->accessBarrier; + return barrier->indexableDataDisplacement(vmThread, src, dst); + +} /* TODO: After all array accesses in the VM have been made arraylet safe, * it should be possible to delete this method + its associated ENVY and diff --git a/runtime/gc_base/gc_internal.h b/runtime/gc_base/gc_internal.h index 734475bc51c..df2f25aebf9 100644 --- a/runtime/gc_base/gc_internal.h +++ b/runtime/gc_base/gc_internal.h @@ -101,6 +101,7 @@ extern J9_CFUNC BOOLEAN j9gc_hot_reference_field_required(J9JavaVM *javaVM); extern J9_CFUNC BOOLEAN j9gc_off_heap_allocation_enabled(J9JavaVM *javaVM); extern J9_CFUNC uint32_t j9gc_max_hot_field_list_length(J9JavaVM *javaVM); extern J9_CFUNC void j9gc_objaccess_indexableStoreAddress(J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, void *value, UDATA isVolatile); +extern J9_CFUNC IDATA j9gc_objaccess_indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst); extern J9_CFUNC void j9gc_objaccess_mixedObjectStoreAddress(J9VMThread *vmThread, j9object_t destObject, UDATA offset, void *value, UDATA isVolatile); extern J9_CFUNC void j9gc_objaccess_cloneObject(J9VMThread *vmThread, j9object_t srcObject, j9object_t destObject); extern J9_CFUNC void j9gc_objaccess_copyObjectFields(J9VMThread *vmThread, J9Class *valueClass, J9Object *srcObject, UDATA srcOffset, J9Object *destObject, UDATA destOffset, MM_objectMapFunction objectMapFunction, void *objectMapData, UDATA initializeLockWord); diff --git a/runtime/gc_glue_java/ArrayletObjectModel.cpp b/runtime/gc_glue_java/ArrayletObjectModel.cpp index 9aa8470f7c7..39536cdf8bd 100644 --- a/runtime/gc_glue_java/ArrayletObjectModel.cpp +++ b/runtime/gc_glue_java/ArrayletObjectModel.cpp @@ -154,6 +154,22 @@ GC_ArrayletObjectModel::fixupInternalLeafPointersAfterCopy(J9IndexableObject *de } #if defined(J9VM_ENV_DATA64) +bool +GC_ArrayletObjectModel::isDataAdjacentToHeader(J9IndexableObject *arrayPtr) +{ + uintptr_t dataSizeInBytes = getDataSizeInBytes(arrayPtr); + return isDataAdjacentToHeader(dataSizeInBytes); +} + +bool +GC_ArrayletObjectModel::isDataAdjacentToHeader(uintptr_t dataSizeInBytes) +{ + MM_GCExtensionsBase *extensions = MM_GCExtensionsBase::getExtensions(_omrVM); + uintptr_t minimumSpineSizeAfterGrowing = extensions->getObjectAlignmentInBytes(); + return ((UDATA_MAX == _largestDesirableArraySpineSize) + || (dataSizeInBytes <= (_largestDesirableArraySpineSize - minimumSpineSizeAfterGrowing - contiguousIndexableHeaderSize()))); +} + void GC_ArrayletObjectModel::AssertArrayPtrIsIndexable(J9IndexableObject *arrayPtr) { diff --git a/runtime/gc_glue_java/ArrayletObjectModel.hpp b/runtime/gc_glue_java/ArrayletObjectModel.hpp index 1d7547fb907..e6ca7aee4ec 100644 --- a/runtime/gc_glue_java/ArrayletObjectModel.hpp +++ b/runtime/gc_glue_java/ArrayletObjectModel.hpp @@ -1307,6 +1307,26 @@ class GC_ArrayletObjectModel : public GC_ArrayletObjectModelBase */ void fixupInternalLeafPointersAfterCopy(J9IndexableObject *destinationPtr, J9IndexableObject *sourcePtr); +#if defined(J9VM_ENV_DATA64) + /** + * Used to determine if the array data is adjacent to its header or in offheap. + * The determination is based on the actual value of dataAddr field in the header. + * + * @param arrayPtr Pointer to the indexable object + * @return true if the arraylet data is adjacent to the header, false otherwise + */ + bool isDataAdjacentToHeader(J9IndexableObject *arrayPtr); + + /** + * Used to determine if the array data should be adjacent to its header or in offheap. + * The determination is based on the size, same how it would be done during the allocation of an object of such a size. + * + * @param dataSizeInBytes the size of data in an indexable object, in bytes, including leaves and alignment padding + * @return true if based on the value of dataSizeInBytes, the arraylet data is adjacent to the header, false otherwise + */ + bool isDataAdjacentToHeader(uintptr_t dataSizeInBytes); +#endif /* defined(J9VM_ENV_DATA64) */ + /** * Initialize the receiver, a new instance of GC_ObjectModel * diff --git a/runtime/gc_realtime/RealtimeAccessBarrier.hpp b/runtime/gc_realtime/RealtimeAccessBarrier.hpp index 274207052a4..fbfcc246816 100644 --- a/runtime/gc_realtime/RealtimeAccessBarrier.hpp +++ b/runtime/gc_realtime/RealtimeAccessBarrier.hpp @@ -156,6 +156,13 @@ class MM_RealtimeAccessBarrier : public MM_ObjectAccessBarrier virtual I_32 backwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots); virtual I_32 forwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots); + virtual IDATA + indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) + { + Assert_MM_unreachable(); + return 0; + } + virtual bool checkStringConstantsLive(J9JavaVM *javaVM, j9object_t stringOne, j9object_t stringTwo); virtual bool checkStringConstantLive(J9JavaVM *javaVM, j9object_t string); diff --git a/runtime/gc_vlhgc/VLHGCAccessBarrier.cpp b/runtime/gc_vlhgc/VLHGCAccessBarrier.cpp index a3df59931c3..7d4c6eee6e5 100644 --- a/runtime/gc_vlhgc/VLHGCAccessBarrier.cpp +++ b/runtime/gc_vlhgc/VLHGCAccessBarrier.cpp @@ -271,6 +271,19 @@ MM_VLHGCAccessBarrier::copyArrayCritical(J9VMThread *vmThread, GC_ArrayObjectMod } } +IDATA +MM_VLHGCAccessBarrier::indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) +{ + IDATA displacement = 0; + + Assert_MM_true(vmThread->isVirtualLargeObjectHeapEnabled); + /* Adjacency check against dst object since src object may be overwritten during sliding compaction. */ + if (_extensions->indexableObjectModel.isDataAdjacentToHeader(dst)) { + displacement = MM_ObjectAccessBarrier::indexableDataDisplacement(vmThread, src, dst); + } + return displacement; +} + void* MM_VLHGCAccessBarrier::jniGetPrimitiveArrayCritical(J9VMThread* vmThread, jarray array, jboolean *isCopy) { diff --git a/runtime/gc_vlhgc/VLHGCAccessBarrier.hpp b/runtime/gc_vlhgc/VLHGCAccessBarrier.hpp index 1fac62182c2..6ec5ded0fec 100644 --- a/runtime/gc_vlhgc/VLHGCAccessBarrier.hpp +++ b/runtime/gc_vlhgc/VLHGCAccessBarrier.hpp @@ -87,6 +87,8 @@ class MM_VLHGCAccessBarrier : public MM_ObjectAccessBarrier virtual I_32 backwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots); virtual I_32 forwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots); + virtual IDATA indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst); + virtual void* jniGetPrimitiveArrayCritical(J9VMThread* vmThread, jarray array, jboolean *isCopy); virtual void jniReleasePrimitiveArrayCritical(J9VMThread* vmThread, jarray array, void * elems, jint mode); virtual const jchar* jniGetStringCritical(J9VMThread* vmThread, jstring str, jboolean *isCopy); diff --git a/runtime/oti/j9nonbuilder.h b/runtime/oti/j9nonbuilder.h index 289f2384fd5..ccbc0ec905c 100644 --- a/runtime/oti/j9nonbuilder.h +++ b/runtime/oti/j9nonbuilder.h @@ -4693,6 +4693,7 @@ typedef struct J9MemoryManagerFunctions { #endif /* !defined(J9VM_ENV_DATA64) */ void ( *j9gc_objaccess_indexableStoreObject)(struct J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, j9object_t value, UDATA isVolatile) ; void ( *j9gc_objaccess_indexableStoreAddress)(struct J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, void *value, UDATA isVolatile) ; + IDATA ( *j9gc_objaccess_indexableDataDisplacement)(struct J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) ; IDATA ( *j9gc_objaccess_mixedObjectReadI32)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ; UDATA ( *j9gc_objaccess_mixedObjectReadU32)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ; I_64 ( *j9gc_objaccess_mixedObjectReadI64)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ;