diff --git a/src/Makefile b/src/Makefile index 545e4e4ecb2e8..c1c0d350f36c3 100644 --- a/src/Makefile +++ b/src/Makefile @@ -48,7 +48,7 @@ SRCS := \ jltypes gf typemap smallintset ast builtins module interpreter symbol \ dlload sys init task array staticdata toplevel jl_uv datatype \ simplevector runtime_intrinsics precompile jloptions mtarraylist \ - threading partr stackwalk gc-common gc gc-debug gc-pages gc-stacks gc-alloc-profiler method \ + threading partr stackwalk gc-common gc gc-debug gc-pages gc-stacks gc-alloc-profiler gc-page-profiler method \ mmtk-gc jlapi signal-handling safepoint timing subtype rtutils gc-heap-snapshot \ crc32c APInt-C processor ircode opaque_closure codegen-stubs coverage runtime_ccall diff --git a/src/gc-page-profiler.c b/src/gc-page-profiler.c index 5af1c3d014770..1c189af01aa0f 100644 --- a/src/gc-page-profiler.c +++ b/src/gc-page-profiler.c @@ -1,5 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license +#ifndef MMTK_GC + #include "gc-page-profiler.h" #ifdef __cplusplus @@ -165,3 +167,5 @@ JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream) #ifdef __cplusplus } #endif + +#endif // !MMTK_GC diff --git a/src/gc.c b/src/gc.c index d88cb0ff2dea9..74a720260e7d5 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3583,6 +3583,16 @@ JL_DLLEXPORT void jl_gc_wb1_noinline(const void *parent) JL_NOTSAFEPOINT jl_unreachable(); } +JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT +{ + jl_unreachable(); +} + +JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT +{ + jl_unreachable(); +} + JL_DLLEXPORT void jl_gc_wb2_noinline(const void *parent, const void *ptr) JL_NOTSAFEPOINT { jl_unreachable(); diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 02c39caf29134..54699ca46c78f 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -191,6 +191,8 @@ XX(jl_gc_pool_alloc_instrumented) \ XX(jl_gc_queue_multiroot) \ XX(jl_gc_queue_root) \ + XX(jl_gc_preserve_begin_hook) \ + XX(jl_gc_preserve_end_hook) \ XX(jl_gc_wb1_noinline) \ XX(jl_gc_wb2_noinline) \ XX(jl_gc_wb_binding_noinline) \ diff --git a/src/julia.h b/src/julia.h index b1a4a62b2ada9..b23f2bc561502 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2110,6 +2110,11 @@ typedef struct _jl_task_t { int8_t threadpoolid; // saved gc stack top for context switches jl_gcframe_t *gcstack; +#ifdef MMTK_GC + // GC stack of objects from gc preserve regions + // These must always be transitively pinned + jl_gcframe_t *gcpreserve_stack; +#endif size_t world_age; // quick lookup for current ptls jl_ptls_t ptls; // == jl_all_tls_states[tid] diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 990bd92f3b499..74e69d9d6fa9f 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -52,6 +52,8 @@ struct FinalLowerGC: private JuliaPassContext { Function *bigAllocFunc; Function *allocTypedFunc; #ifdef MMTK_GC + Function *gcPreserveBeginHookFunc; + Function *gcPreserveEndHookFunc; Function *writeBarrier1Func; Function *writeBarrier2Func; Function *writeBarrierBindingFunc; @@ -145,7 +147,7 @@ void FinalLowerGC::lowerPushGCFrame(CallInst *target, Function &F) IRBuilder<> builder(target->getContext()); builder.SetInsertPoint(&*(++BasicBlock::iterator(target))); StoreInst *inst = builder.CreateAlignedStore( - ConstantInt::get(getSizeTy(F.getContext()), JL_GC_ENCODE_PUSHARGS(nRoots)), + ConstantInt::get(getSizeTy(F.getContext()), JL_GC_ENCODE_PUSHARGS_NO_TPIN(nRoots)), builder.CreateBitCast( builder.CreateConstInBoundsGEP1_32(T_prjlvalue, gcframe, 0), getSizeTy(F.getContext())->getPointerTo()), @@ -407,12 +409,14 @@ bool FinalLowerGC::doInitialization(Module &M) { bigAllocFunc = getOrDeclare(jl_well_known::GCBigAlloc); allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped); #ifdef MMTK_GC + gcPreserveBeginHookFunc = getOrDeclare(jl_well_known::GCPreserveBeginHook); + gcPreserveEndHookFunc = getOrDeclare(jl_well_known::GCPreserveEndHook); writeBarrier1Func = getOrDeclare(jl_well_known::GCWriteBarrier1); writeBarrier2Func = getOrDeclare(jl_well_known::GCWriteBarrier2); writeBarrierBindingFunc = getOrDeclare(jl_well_known::GCWriteBarrierBinding); writeBarrier1SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier1Slow); writeBarrier2SlowFunc = getOrDeclare(jl_well_known::GCWriteBarrier2Slow); - GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc}; + GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, gcPreserveBeginHookFunc, gcPreserveEndHookFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc}; #else GlobalValue *functionList[] = {queueRootFunc, queueBindingFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc}; #endif @@ -432,8 +436,8 @@ bool FinalLowerGC::doInitialization(Module &M) { bool FinalLowerGC::doFinalization(Module &M) { #ifdef MMTK_GC - GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc}; - queueRootFunc = poolAllocFunc = bigAllocFunc = writeBarrier1Func = writeBarrier2Func = writeBarrierBindingFunc = writeBarrier1SlowFunc = writeBarrier2SlowFunc = nullptr; + GlobalValue *functionList[] = {queueRootFunc, poolAllocFunc, bigAllocFunc, gcPreserveBeginHookFunc, gcPreserveEndHookFunc, writeBarrier1Func, writeBarrier2Func, writeBarrierBindingFunc, writeBarrier1SlowFunc, writeBarrier2SlowFunc}; + queueRootFunc = poolAllocFunc = bigAllocFunc = gcPreserveBeginHookFunc = gcPreserveEndHookFunc = writeBarrier1Func = writeBarrier2Func = writeBarrierBindingFunc = writeBarrier1SlowFunc = writeBarrier2SlowFunc = nullptr; #else GlobalValue *functionList[] = {queueRootFunc, queueBindingFunc, poolAllocFunc, bigAllocFunc, allocTypedFunc}; queueRootFunc = queueBindingFunc = poolAllocFunc = bigAllocFunc = allocTypedFunc = nullptr; diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index b76f4c38227f2..a6bacff36323f 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -2308,9 +2308,58 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) { continue; } Value *callee = CI->getCalledOperand(); +#ifdef MMTK_GC + if (callee && (callee == gc_flush_func)) { + /* No replacement */ + } else if (callee && (callee == gc_preserve_begin_func)) { + /* Replace with a call to the hook functions */ + // Initialize an IR builder. + IRBuilder<> builder(CI); + + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + size_t nargs = 0; + State S2(F); + + std::vector args; + for (Use &U : CI->args()) { + Value *V = U; + if (isa(V)) + continue; + if (isa(V->getType())) { + if (isSpecialPtr(V->getType())) { + int Num = Number(S2, V); + if (Num >= 0) { + nargs++; + Value *Val = GetPtrForNumber(S2, Num, CI); + args.push_back(Val); + } + } + } else { + std::vector Nums = NumberAll(S2, V); + for (int Num : Nums) { + if (Num < 0) + continue; + Value *Val = GetPtrForNumber(S2, Num, CI); + args.push_back(Val); + nargs++; + } + } + } + args.insert(args.begin(), ConstantInt::get(T_size, nargs)); + + ArrayRef args_llvm = ArrayRef(args); + builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveBeginHook), args_llvm ); + } else if (callee && (callee == gc_preserve_end_func)) { + /* Replace with a call to the hook functions */ + // Initialize an IR builder. + IRBuilder<> builder(CI); + builder.SetCurrentDebugLocation(CI->getDebugLoc()); + builder.CreateCall(getOrDeclare(jl_well_known::GCPreserveEndHook), {}); +#else if (callee && (callee == gc_flush_func || callee == gc_preserve_begin_func || callee == gc_preserve_end_func)) { /* No replacement */ +#endif } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) { auto *obj = CI->getOperand(0); auto *ASCI = new AddrSpaceCastInst(obj, JuliaType::get_pjlvalue_ty(obj->getContext()), "", CI); diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index 73d765f61e856..8d2e0c2d14ca5 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -334,6 +334,8 @@ namespace jl_well_known { static const char *GC_QUEUE_BINDING_NAME = XSTR(jl_gc_queue_binding); static const char *GC_ALLOC_TYPED_NAME = XSTR(jl_gc_alloc_typed); #ifdef MMTK_GC + static const char *GC_PRESERVE_BEGIN_HOOK_NAME = XSTR(jl_gc_preserve_begin_hook); + static const char *GC_PRESERVE_END_HOOK_NAME = XSTR(jl_gc_preserve_end_hook); static const char *GC_WB_1_NAME = XSTR(jl_gc_wb1_noinline); static const char *GC_WB_2_NAME = XSTR(jl_gc_wb2_noinline); static const char *GC_WB_BINDING_NAME = XSTR(jl_gc_wb_binding_noinline); @@ -424,6 +426,34 @@ namespace jl_well_known { }); #ifdef MMTK_GC + const WellKnownFunctionDescription GCPreserveBeginHook( + GC_PRESERVE_BEGIN_HOOK_NAME, + [](const JuliaPassContext &context) { + auto func = Function::Create( + FunctionType::get( + Type::getVoidTy(context.getLLVMContext()), + { T_size_t(context) }, + true), + Function::ExternalLinkage, + GC_PRESERVE_BEGIN_HOOK_NAME); + + func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + return func; + }); + + const WellKnownFunctionDescription GCPreserveEndHook( + GC_PRESERVE_END_HOOK_NAME, + [](const JuliaPassContext &context) { + auto func = Function::Create( + FunctionType::get( + Type::getVoidTy(context.getLLVMContext()), + { }, + false), + Function::ExternalLinkage, + GC_PRESERVE_END_HOOK_NAME); + func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly); + return func; + }); const WellKnownFunctionDescription GCWriteBarrier1( GC_WB_1_NAME, [](const JuliaPassContext &context) { diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index 21aaed8e9ba75..1e672433be596 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -134,6 +134,8 @@ namespace jl_intrinsics { extern const IntrinsicDescription safepoint; #ifdef MMTK_GC + extern const IntrinsicDescription gcPreserveBeginHook; + extern const IntrinsicDescription gcPreserveEndHook; extern const IntrinsicDescription writeBarrier1; extern const IntrinsicDescription writeBarrier2; extern const IntrinsicDescription writeBarrierBinding; @@ -168,6 +170,8 @@ namespace jl_well_known { extern const WellKnownFunctionDescription GCAllocTyped; #ifdef MMTK_GC + extern const WellKnownFunctionDescription GCPreserveBeginHook; + extern const WellKnownFunctionDescription GCPreserveEndHook; extern const WellKnownFunctionDescription GCWriteBarrier1; extern const WellKnownFunctionDescription GCWriteBarrier2; extern const WellKnownFunctionDescription GCWriteBarrierBinding; diff --git a/src/mmtk-gc.c b/src/mmtk-gc.c index 427495ea0aa64..e0363c8e70386 100644 --- a/src/mmtk-gc.c +++ b/src/mmtk-gc.c @@ -570,6 +570,53 @@ JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array mmtk_memory_region_copy(&ptls->mmtk_mutator, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n); } +#define jl_p_gcpreserve_stack (jl_current_task->gcpreserve_stack) + +// This macro currently uses malloc instead of alloca because this function will exit +// after pushing the roots into the gc_preserve_stack, which means that the preserve_begin function's +// stack frame will be destroyed (together with its alloca variables). When we support lowering this code +// inside the same function that is doing the preserve_begin/preserve_end calls we should be able to simple use allocas. +// Note also that we use a separate stack for gc preserve roots to avoid the possibility of calling free +// on a stack that has been allocated with alloca instead of malloc, which could happen depending on the order in which +// JL_GC_POP() and jl_gc_preserve_end_hook() occurs. + +#define JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(rts_var,n) \ + rts_var = ((jl_value_t**)malloc(((n)+2)*sizeof(jl_value_t*)))+2; \ + ((void**)rts_var)[-2] = (void*)JL_GC_ENCODE_PUSHARGS(n); \ + ((void**)rts_var)[-1] = jl_p_gcpreserve_stack; \ + memset((void*)rts_var, 0, (n)*sizeof(jl_value_t*)); \ + jl_p_gcpreserve_stack = (jl_gcframe_t*)&(((void**)rts_var)[-2]); \ + +#define JL_GC_POP_PRESERVE_ROOT_OBJS() \ + jl_gcframe_t *curr = jl_p_gcpreserve_stack; \ + if(curr) { \ + (jl_p_gcpreserve_stack = jl_p_gcpreserve_stack->prev); \ + free(curr); \ + } + +// Add each argument as a tpin root object. +// However, we cannot use JL_GC_PUSH and JL_GC_POP since the slots should live +// beyond this function. Instead, we maintain a tpin stack by mallocing/freeing +// the frames for each of the preserve regions we encounter +JL_DLLEXPORT void jl_gc_preserve_begin_hook(int n, ...) JL_NOTSAFEPOINT +{ + jl_value_t** frame; + JL_GC_PUSHARGS_PRESERVE_ROOT_OBJS(frame, n); + if (n == 0) return; + + va_list args; + va_start(args, n); + for (int i = 0; i < n; i++) { + frame[i] = va_arg(args, jl_value_t *); + } + va_end(args); +} + +JL_DLLEXPORT void jl_gc_preserve_end_hook(void) JL_NOTSAFEPOINT +{ + JL_GC_POP_PRESERVE_ROOT_OBJS(); +} + // No inline write barrier -- only used for debugging JL_DLLEXPORT void jl_gc_wb1_noinline(const void *parent) JL_NOTSAFEPOINT {