Merge pull request #20564 from LinHu2016/off-heap-incremental4

New vm/gc API j9gc_objaccess_indexableDataDisplacement

runtime/gc/gctable.c

@@ -143,6 +143,7 @@ J9MemoryManagerFunctions MemoryManagerFunctions = {
 #endif /* !J9VM_ENV_DATA64 */
 	j9gc_objaccess_indexableStoreObject,
 	j9gc_objaccess_indexableStoreAddress,
+	j9gc_objaccess_indexableDataDisplacement,
 	j9gc_objaccess_mixedObjectReadI32,
 	j9gc_objaccess_mixedObjectReadU32,
 	j9gc_objaccess_mixedObjectReadI64,

runtime/gc_base/ObjectAccessBarrier.hpp

@@ -226,6 +226,10 @@ class MM_ObjectAccessBarrier : public MM_BaseVirtual
 	virtual void staticStoreU64(J9VMThread *vmThread, J9Class *clazz, U_64 *destSlot, U_64 value, bool isVolatile=false);
 	virtual void staticStoreI64(J9VMThread *vmThread, J9Class *clazz, I_64 *destSlot, I_64 value, bool isVolatile=false);
 
+	MMINLINE virtual IDATA indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst)
+	{
+		return (IDATA)(((UDATA)dst) - ((UDATA)src));
+	}
 	virtual U_8 *getArrayObjectDataAddress(J9VMThread *vmThread, J9IndexableObject *arrayObject);
 	virtual j9objectmonitor_t *getLockwordAddress(J9VMThread *vmThread, J9Object *object);
 	virtual void cloneObject(J9VMThread *vmThread, J9Object *srcObject, J9Object *destObject);

runtime/gc_base/accessBarrier.cpp

@@ -341,6 +341,24 @@ j9gc_objaccess_staticStoreU64Split(J9VMThread *vmThread, J9Class *clazz, U_64 *d
 	barrier->staticStoreU64(vmThread, clazz, destSlot, value64, 0 != isVolatile);
 }
 
+/**
+ * Returns the displacement for the data of moved array object.
+ * Used by the JIT, should only be called for off heap enabled cases,
+ * For adjacent Array, displacement = dst - src
+ * For Off-heap Array, displacement = 0.
+ *
+ * @param vmThread Pointer to the current J9VMThread
+ * @param src Pointer to the array object before moving
+ * @param dst Pointer to the array object after moving
+ * @return displacement
+ */
+IDATA
+j9gc_objaccess_indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst)
+{
+	MM_ObjectAccessBarrier *barrier = MM_GCExtensions::getExtensions(vmThread)->accessBarrier;
+	return barrier->indexableDataDisplacement(vmThread, src, dst);
+
+}
 
 /* TODO: After all array accesses in the VM have been made arraylet safe, 
  * it should be possible to delete this method + its associated ENVY and 

runtime/gc_base/gc_internal.h

@@ -101,6 +101,7 @@ extern J9_CFUNC BOOLEAN j9gc_hot_reference_field_required(J9JavaVM *javaVM);
 extern J9_CFUNC BOOLEAN j9gc_off_heap_allocation_enabled(J9JavaVM *javaVM);
 extern J9_CFUNC uint32_t j9gc_max_hot_field_list_length(J9JavaVM *javaVM);
 extern J9_CFUNC void j9gc_objaccess_indexableStoreAddress(J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, void *value, UDATA isVolatile);
+extern J9_CFUNC IDATA j9gc_objaccess_indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst);
 extern J9_CFUNC void j9gc_objaccess_mixedObjectStoreAddress(J9VMThread *vmThread, j9object_t destObject, UDATA offset, void *value, UDATA isVolatile);
 extern J9_CFUNC void j9gc_objaccess_cloneObject(J9VMThread *vmThread, j9object_t srcObject, j9object_t destObject);
 extern J9_CFUNC void j9gc_objaccess_copyObjectFields(J9VMThread *vmThread, J9Class *valueClass, J9Object *srcObject, UDATA srcOffset, J9Object *destObject, UDATA destOffset, MM_objectMapFunction objectMapFunction, void *objectMapData, UDATA initializeLockWord);

runtime/gc_glue_java/ArrayletObjectModel.cpp

@@ -154,6 +154,22 @@ GC_ArrayletObjectModel::fixupInternalLeafPointersAfterCopy(J9IndexableObject *de
 }
 
 #if defined(J9VM_ENV_DATA64)
+bool
+GC_ArrayletObjectModel::isDataAdjacentToHeader(J9IndexableObject *arrayPtr)
+{
+	uintptr_t dataSizeInBytes = getDataSizeInBytes(arrayPtr);
+	return isDataAdjacentToHeader(dataSizeInBytes);
+}
+
+bool
+GC_ArrayletObjectModel::isDataAdjacentToHeader(uintptr_t dataSizeInBytes)
+{
+	MM_GCExtensionsBase *extensions = MM_GCExtensionsBase::getExtensions(_omrVM);
+	uintptr_t minimumSpineSizeAfterGrowing = extensions->getObjectAlignmentInBytes();
+	return ((UDATA_MAX == _largestDesirableArraySpineSize)
+			|| (dataSizeInBytes <= (_largestDesirableArraySpineSize - minimumSpineSizeAfterGrowing - contiguousIndexableHeaderSize())));
+}
+
 void
 GC_ArrayletObjectModel::AssertArrayPtrIsIndexable(J9IndexableObject *arrayPtr)
 {

runtime/gc_glue_java/ArrayletObjectModel.hpp

@@ -1307,6 +1307,26 @@ class GC_ArrayletObjectModel : public GC_ArrayletObjectModelBase
 	 */
 	void fixupInternalLeafPointersAfterCopy(J9IndexableObject *destinationPtr, J9IndexableObject *sourcePtr);
 
+#if defined(J9VM_ENV_DATA64)
+	/**
+	 * Used to determine if the array data is adjacent to its header or in offheap.
+	 * The determination is based on the actual value of dataAddr field in the header.
+	 *
+	 * @param arrayPtr Pointer to the indexable object
+	 * @return true if the arraylet data is adjacent to the header, false otherwise
+	 */
+	bool isDataAdjacentToHeader(J9IndexableObject *arrayPtr);
+
+	/**
+	 * Used to determine if the array data should be adjacent to its header or in offheap.
+	 * The determination is based on the size, same how it would be done during the allocation of an object of such a size.
+	 *
+	 * @param dataSizeInBytes the size of data in an indexable object, in bytes, including leaves and alignment padding
+	 * @return true if based on the value of dataSizeInBytes, the arraylet data is adjacent to the header, false otherwise
+	 */
+	bool isDataAdjacentToHeader(uintptr_t dataSizeInBytes);
+#endif /* defined(J9VM_ENV_DATA64) */
+
 	/**
 	 * Initialize the receiver, a new instance of GC_ObjectModel
 	 * 

runtime/gc_realtime/RealtimeAccessBarrier.hpp

@@ -156,6 +156,13 @@ class MM_RealtimeAccessBarrier : public MM_ObjectAccessBarrier
 	virtual I_32 backwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots);
 	virtual I_32 forwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots);
 
+	virtual IDATA
+	indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst)
+	{
+		Assert_MM_unreachable();
+		return 0;
+	}
+
 	virtual bool checkStringConstantsLive(J9JavaVM *javaVM, j9object_t stringOne, j9object_t stringTwo);
 	virtual bool checkStringConstantLive(J9JavaVM *javaVM, j9object_t string);
 

runtime/gc_vlhgc/VLHGCAccessBarrier.cpp

@@ -271,6 +271,19 @@ MM_VLHGCAccessBarrier::copyArrayCritical(J9VMThread *vmThread, GC_ArrayObjectMod
 	}
 }
 
+IDATA
+MM_VLHGCAccessBarrier::indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst)
+{
+	IDATA displacement = 0;
+
+	Assert_MM_true(vmThread->isVirtualLargeObjectHeapEnabled);
+	/* Adjacency check against dst object since src object may be overwritten during sliding compaction. */
+	if (_extensions->indexableObjectModel.isDataAdjacentToHeader(dst)) {
+		displacement = MM_ObjectAccessBarrier::indexableDataDisplacement(vmThread, src, dst);
+	}
+	return displacement;
+}
+
 void*
 MM_VLHGCAccessBarrier::jniGetPrimitiveArrayCritical(J9VMThread* vmThread, jarray array, jboolean *isCopy)
 {

runtime/gc_vlhgc/VLHGCAccessBarrier.hpp

@@ -87,6 +87,8 @@ class MM_VLHGCAccessBarrier : public MM_ObjectAccessBarrier
 	virtual I_32 backwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots);
 	virtual I_32 forwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots);
 
+	virtual IDATA indexableDataDisplacement(J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst);
+
 	virtual void* jniGetPrimitiveArrayCritical(J9VMThread* vmThread, jarray array, jboolean *isCopy);
 	virtual void jniReleasePrimitiveArrayCritical(J9VMThread* vmThread, jarray array, void * elems, jint mode);
 	virtual const jchar* jniGetStringCritical(J9VMThread* vmThread, jstring str, jboolean *isCopy);

runtime/oti/j9nonbuilder.h

@@ -4693,6 +4693,7 @@ typedef struct J9MemoryManagerFunctions {
 #endif /* !defined(J9VM_ENV_DATA64) */
 	void  ( *j9gc_objaccess_indexableStoreObject)(struct J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, j9object_t value, UDATA isVolatile) ;
 	void  ( *j9gc_objaccess_indexableStoreAddress)(struct J9VMThread *vmThread, J9IndexableObject *destObject, I_32 index, void *value, UDATA isVolatile) ;
+	IDATA  ( *j9gc_objaccess_indexableDataDisplacement)(struct J9VMThread *vmThread, J9IndexableObject *src, J9IndexableObject *dst) ;
 	IDATA  ( *j9gc_objaccess_mixedObjectReadI32)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ;
 	UDATA  ( *j9gc_objaccess_mixedObjectReadU32)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ;
 	I_64  ( *j9gc_objaccess_mixedObjectReadI64)(struct J9VMThread *vmThread, j9object_t srcObject, UDATA offset, UDATA isVolatile) ;