Process gc preserve

src/Makefile

@@ -48,7 +48,7 @@ SRCS := \
 	jltypes gf typemap smallintset ast builtins module interpreter symbol \
 	dlload sys init task array staticdata toplevel jl_uv datatype \
 	simplevector runtime_intrinsics precompile jloptions mtarraylist \
-	threading partr stackwalk gc-common gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \
+	threading partr stackwalk gc-common gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \
 	mmtk-gc jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \
 	crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall
 

src/gc.c

@@ -3583,6 +3583,16 @@ JL_DLLEXPORT void jl_gc_wb1_noinline(const void *parent) JL_NOTSAFEPOINT
     jl_unreachable();
 }
 
+JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT
+{
+    jl_unreachable();
+}
+
+JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT
+{
+    jl_unreachable();
+}
+
 JL_DLLEXPORT void jl_gc_wb2_noinline(const void *parent, const void *ptr) JL_NOTSAFEPOINT
 {
     jl_unreachable();

src/jl_exported_funcs.inc

@@ -191,6 +191,8 @@
     XX(jl_gc_pool_alloc_instrumented) \
     XX(jl_gc_queue_multiroot) \
     XX(jl_gc_queue_root) \
+    XX(jl_gc_preserve_begin_hook) \
+    XX(jl_gc_preserve_end_hook) \
     XX(jl_gc_wb1_noinline) \
     XX(jl_gc_wb2_noinline) \
     XX(jl_gc_wb_binding_noinline) \

src/julia.h

@@ -2110,6 +2110,11 @@ typedef struct _jl_task_t {
     int8_t threadpoolid;
     // saved gc stack top for context switches
     jl_gcframe_t *gcstack;
+#ifdef MMTK_GC
+    // GC stack of objects from gc preserve regions
+    // These must always be transitively pinned
+    jl_gcframe_t *gcpreserve_stack;
+#endif
     size_t world_age;
     // quick lookup for current ptls
     jl_ptls_t ptls; // == jl_all_tls_states[tid]

src/llvm-final-gc-lowering.cpp

@@ -52,6 +52,8 @@ struct FinalLowerGC: private JuliaPassContext {
     Function *bigAllocFunc;
     Function *allocTypedFunc;
 #ifdef MMTK_GC
+    Function *gcPreserveBeginHookFunc;
+    Function *gcPreserveEndHookFunc;
     Function *writeBarrier1Func;
     Function *writeBarrier2Func;
     Function *writeBarrierBindingFunc;
@@ -145,7 +147,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F)
     IRBuilder<> builder(target->getContext());
     builder.SetInsertPoint(&*(++BasicBlock::iterator(target)));
     StoreInst *inst = builder.CreateAlignedStore(
-                ConstantInt::get(getSizeTy(F.getContext()), JL_GC_ENCODE_PUSHARGS(nRoots)),
+                ConstantInt::get(getSizeTy(F.getContext()), JL_GC_ENCODE_PUSHARGS_NO_TPIN(nRoots)),
                 builder.CreateBitCast(
                         builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0),
                         getSizeTy(F.getContext())->getPointerTo()),
@@ -407,12 +409,14 @@ bool FinalLowerGC::doInitialization(Module &M) {
     bigAllocFunc = getOrDeclare(jl_well_known::GCBigAlloc);
     allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
 #ifdef MMTK_GC
+    gcPreserveBeginHookFunc = getOrDeclare(jl_well_known::GCPreserveBeginHook);
+    gcPreserveEndHookFunc = getOrDeclare(jl_well_known::GCPreserveEndHook);
     writeBarrier1Func = getOrDeclare(jl_well_known::GCWriteBarrier1);
     writeBarrier2Func = getOrDeclare(jl_well_known::GCWriteBarrier2);
     writeBarrierBindingFunc = getOrDeclare(jl_well_known::GCWriteBarrierBinding);
     writeBarrier1SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier1Slow);
     writeBarrier2SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier2Slow);
-    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc};
+    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, gcPreserveBeginHookFunc, gcPreserveEndHookFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc};
 #else
     GlobalValue *functionList[] = {queueRootFunc, queueBindingFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc};
 #endif
@@ -432,8 +436,8 @@ bool FinalLowerGC::doInitialization(Module &M) {
 bool FinalLowerGC::doFinalization(Module &M)
 {
 #ifdef MMTK_GC
-    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc};
-    queueRootFunc = poolAllocFunc = bigAllocFunc = writeBarrier1Func = writeBarrier2Func = writeBarrierBindingFunc = writeBarrier1SlowFunc = writeBarrier2SlowFunc = nullptr;
+    GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, gcPreserveBeginHookFunc, gcPreserveEndHookFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc};
+    queueRootFunc = poolAllocFunc = bigAllocFunc = gcPreserveBeginHookFunc = gcPreserveEndHookFunc = writeBarrier1Func = writeBarrier2Func = writeBarrierBindingFunc = writeBarrier1SlowFunc = writeBarrier2SlowFunc = nullptr;
 #else
     GlobalValue *functionList[] = {queueRootFunc, queueBindingFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc};
     queueRootFunc = queueBindingFunc = poolAllocFunc = bigAllocFunc = allocTypedFunc = nullptr;

src/llvm-late-gc-lowering.cpp

@@ -2308,9 +2308,58 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) {
                 continue;
             }
             Value *callee = CI->getCalledOperand();
+#ifdef MMTK_GC
+            if (callee && (callee == gc_flush_func)) {
+                /* No replacement */
+            } else if (callee && (callee == gc_preserve_begin_func)) {
+                /* Replace with a call to the hook functions */
+                // Initialize an IR builder.
+                IRBuilder<> builder(CI);
+
+                builder.SetCurrentDebugLocation(CI->getDebugLoc());
+                size_t nargs = 0;
+                State S2(F);
+
+                std::vector<Value*> args;
+                for (Use &U : CI->args()) {
+                    Value *V = U;
+                    if (isa<Constant>(V))
+                        continue;
+                    if (isa<PointerType>(V->getType())) {
+                        if (isSpecialPtr(V->getType())) {
+                            int Num = Number(S2, V);
+                            if (Num >= 0) {
+                                nargs++;
+                                Value *Val = GetPtrForNumber(S2, Num, CI);
+                                args.push_back(Val);
+                            }
+                        }
+                    } else {
+                        std::vector<int> Nums = NumberAll(S2, V);
+                        for (int Num : Nums) {
+                            if (Num < 0)
+                                continue;
+                            Value *Val = GetPtrForNumber(S2, Num, CI);
+                            args.push_back(Val);
+                            nargs++;
+                        }
+                    }
+                }
+                args.insert(args.begin(), ConstantInt::get(T_size, nargs));
+
+                ArrayRef<Value*> args_llvm = ArrayRef<Value*>(args);
+                builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveBeginHook), args_llvm );
+            } else if (callee && (callee == gc_preserve_end_func)) {
+                /* Replace with a call to the hook functions */
+                // Initialize an IR builder.
+                IRBuilder<> builder(CI);
+                builder.SetCurrentDebugLocation(CI->getDebugLoc());
+                builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveEndHook), {});
+#else
             if (callee && (callee == gc_flush_func || callee == gc_preserve_begin_func
                         || callee == gc_preserve_end_func)) {
                 /* No replacement */
+#endif
             } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) {
                 auto *obj = CI->getOperand(0);
                 auto *ASCI = new AddrSpaceCastInst(obj, JuliaType::get_pjlvalue_ty(obj->getContext()), "", CI);

src/llvm-pass-helpers.cpp

@@ -334,6 +334,8 @@ namespace jl_well_known {
     static const char *GC_QUEUE_BINDING_NAME = XSTR(jl_gc_queue_binding);
     static const char *GC_ALLOC_TYPED_NAME = XSTR(jl_gc_alloc_typed);
 #ifdef MMTK_GC
+    static const char *GC_PRESERVE_BEGIN_HOOK_NAME = XSTR(jl_gc_preserve_begin_hook);
+    static const char *GC_PRESERVE_END_HOOK_NAME = XSTR(jl_gc_preserve_end_hook);
     static const char *GC_WB_1_NAME = XSTR(jl_gc_wb1_noinline);
     static const char *GC_WB_2_NAME = XSTR(jl_gc_wb2_noinline);
     static const char *GC_WB_BINDING_NAME = XSTR(jl_gc_wb_binding_noinline);
@@ -424,6 +426,34 @@ namespace jl_well_known {
         });
 
 #ifdef MMTK_GC
+    const WellKnownFunctionDescription GCPreserveBeginHook(
+        GC_PRESERVE_BEGIN_HOOK_NAME,
+        [](const JuliaPassContext &context) {
+            auto func = Function::Create(
+                FunctionType::get(
+                    Type::getVoidTy(context.getLLVMContext()),
+                    { T_size_t(context) },
+                    true),
+                Function::ExternalLinkage,
+                GC_PRESERVE_BEGIN_HOOK_NAME);
+
+            func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
+            return func;
+        });
+
+    const WellKnownFunctionDescription GCPreserveEndHook(
+        GC_PRESERVE_END_HOOK_NAME,
+        [](const JuliaPassContext &context) {
+            auto func = Function::Create(
+                FunctionType::get(
+                    Type::getVoidTy(context.getLLVMContext()),
+                    {  },
+                    false),
+                Function::ExternalLinkage,
+                GC_PRESERVE_END_HOOK_NAME);
+            func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
+            return func;
+        });
     const WellKnownFunctionDescription GCWriteBarrier1(
         GC_WB_1_NAME,
         [](const JuliaPassContext &context) {

src/llvm-pass-helpers.h

@@ -134,6 +134,8 @@ namespace jl_intrinsics {
     extern const IntrinsicDescription safepoint;
 
 #ifdef MMTK_GC
+    extern const IntrinsicDescription gcPreserveBeginHook;
+    extern const IntrinsicDescription gcPreserveEndHook;
     extern const IntrinsicDescription writeBarrier1;
     extern const IntrinsicDescription writeBarrier2;
     extern const IntrinsicDescription writeBarrierBinding;
@@ -168,6 +170,8 @@ namespace jl_well_known {
     extern const WellKnownFunctionDescription GCAllocTyped;
 
 #ifdef MMTK_GC
+    extern const WellKnownFunctionDescription GCPreserveBeginHook;
+    extern const WellKnownFunctionDescription GCPreserveEndHook;
     extern const WellKnownFunctionDescription GCWriteBarrier1;
     extern const WellKnownFunctionDescription GCWriteBarrier2;
     extern const WellKnownFunctionDescription GCWriteBarrierBinding;

src/mmtk-gc.c

@@ -570,6 +570,53 @@ JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array
     mmtk_memory_region_copy(&ptls->mmtk_mutator, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n);
 }
 
+#define jl_p_gcpreserve_stack (jl_current_task->gcpreserve_stack)
+
+// This macro currently uses malloc instead of alloca because this function will exit
+// after pushing the roots into the gc_preserve_stack, which means that the preserve_begin function's
+// stack frame will be destroyed (together with its alloca variables). When we support lowering this code
+// inside the same function that is doing the preserve_begin/preserve_end calls we should be able to simple use allocas.
+// Note also that we use a separate stack for gc preserve roots to avoid the possibility of calling free
+// on a stack that has been allocated with alloca instead of malloc, which could happen depending on the order in which
+// JL_GC_POP() and jl_gc_preserve_end_hook() occurs.
+
+#define JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(rts_var,n)                                                    \
+  rts_var = ((jl_value_t**)malloc(((n)+2)*sizeof(jl_value_t*)))+2;                                      \
+  ((void**)rts_var)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(n);                                              \
+  ((void**)rts_var)[-1] = jl_p_gcpreserve_stack;                                                        \
+  memset((void*)rts_var, 0, (n)*sizeof(jl_value_t*));                                                   \
+  jl_p_gcpreserve_stack = (jl_gcframe_t*)&(((void**)rts_var)[-2]);                                      \
+
+#define JL_GC_POP_PRESERVE_ROOT_OBJS()                                                                  \
+    jl_gcframe_t *curr = jl_p_gcpreserve_stack;                                                         \
+    if(curr) {                                                                                          \
+        (jl_p_gcpreserve_stack = jl_p_gcpreserve_stack->prev);                                          \
+        free(curr);                                                                                     \
+    }
+
+// Add each argument as a tpin root object.
+// However, we cannot use JL_GC_PUSH and JL_GC_POP since the slots should live
+// beyond this function. Instead, we maintain a tpin stack by mallocing/freeing
+// the frames for each of the preserve regions we encounter
+JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT
+{
+    jl_value_t** frame;
+    JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(frame, n);
+    if (n == 0) return;
+
+    va_list args;
+    va_start(args, n);
+    for (int i = 0; i < n; i++) {
+        frame[i] = va_arg(args, jl_value_t *);
+    }
+    va_end(args);
+}
+
+JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT
+{
+    JL_GC_POP_PRESERVE_ROOT_OBJS();
+}
+
 // No inline write barrier -- only used for debugging
 JL_DLLEXPORT void jl_gc_wb1_noinline(const void *parent) JL_NOTSAFEPOINT
 {