Adding support for allocating in a non-moving space

src/datatype.c

@@ -63,7 +63,7 @@ JL_DLLEXPORT jl_typename_t *jl_new_typename_in(jl_sym_t *name, jl_module_t *modu
 {
     jl_task_t *ct = jl_current_task;
     jl_typename_t *tn =
-        (jl_typename_t*)jl_gc_alloc(ct->ptls, sizeof(jl_typename_t),
+        (jl_typename_t*)jl_gc_alloc_non_moving(ct->ptls, sizeof(jl_typename_t),
                                     jl_typename_type);
     tn->name = name;
     tn->module = module;
@@ -95,7 +95,7 @@ jl_datatype_t *jl_new_abstracttype(jl_value_t *name, jl_module_t *module, jl_dat
 jl_datatype_t *jl_new_uninitialized_datatype(void)
 {
     jl_task_t *ct = jl_current_task;
-    jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type);
+    jl_datatype_t *t = (jl_datatype_t*)jl_gc_alloc_non_moving(ct->ptls, sizeof(jl_datatype_t), jl_datatype_type);
     jl_set_typetagof(t, jl_datatype_tag, 0);
     t->hash = 0;
     t->hasfreetypevars = 0;

src/gc-common.c

@@ -414,6 +414,12 @@ JL_DLLEXPORT jl_value_t *(jl_gc_alloc)(jl_ptls_t ptls, size_t sz, void *ty)
     return jl_gc_alloc_(ptls, sz, ty);
 }
 
+JL_DLLEXPORT jl_value_t *(jl_gc_alloc_non_moving)(jl_ptls_t ptls, size_t sz, void *ty)
+{
+    return jl_gc_alloc__non_moving(ptls, sz, ty);
+}
+
+
 // Instrumented version of jl_gc_big_alloc_inner, called into by
 // LLVM-generated code.
 JL_DLLEXPORT jl_value_t *jl_gc_big_alloc(jl_ptls_t ptls, size_t sz)

src/julia.h

@@ -2483,6 +2483,23 @@ STATIC_INLINE void mmtk_immortal_post_alloc_fast(MMTkMutatorContext* mutator, vo
     }
 }
 
+STATIC_INLINE void mmtk_non_moving_post_alloc_fast(MMTkMutatorContext* mutator, void* obj, size_t size) {
+    // FIXME: since allocation currently happens in the immortal space we need the same WB code
+    if (MMTK_NEEDS_WRITE_BARRIER == MMTK_OBJECT_BARRIER) {
+        intptr_t addr = (intptr_t) obj;
+        uint8_t* meta_addr = (uint8_t*) (MMTK_SIDE_LOG_BIT_BASE_ADDRESS) + (addr >> 6);
+        intptr_t shift = (addr >> 3) & 0b111;
+        while(1) {
+            uint8_t old_val = *meta_addr;
+            uint8_t new_val = old_val | (1 << shift);
+            if (jl_atomic_cmpswap((_Atomic(uint8_t)*)meta_addr, &old_val, new_val)) {
+                break;
+            }
+        }
+    }
+}
+
+
 #endif
 
 #ifdef __cplusplus

src/julia_internal.h

@@ -341,6 +341,7 @@ jl_value_t *jl_gc_big_alloc_noinline(jl_ptls_t ptls, size_t allocsz);
 #ifdef MMTK_GC
 JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offset, int osize, void* ty);
 JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t allocsz);
+JL_DLLIMPORT jl_value_t *jl_mmtk_gc_alloc_non_moving(jl_ptls_t ptls, int osize, void* ty);
 JL_DLLIMPORT extern void mmtk_post_alloc(void* mutator, void* obj, size_t bytes, int allocator);
 JL_DLLIMPORT extern void mmtk_initialize_collection(void* tls);
 #endif // MMTK_GC
@@ -506,6 +507,20 @@ STATIC_INLINE jl_value_t *jl_gc_alloc_(jl_ptls_t ptls, size_t sz, void *ty)
     maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);
     return v;
 }
+
+STATIC_INLINE jl_value_t *jl_gc_alloc__non_moving(jl_ptls_t ptls, size_t sz, void *ty)
+{
+    jl_value_t *v;
+    const size_t allocsz = sz + sizeof(jl_taggedvalue_t);
+
+    if (allocsz < sz) // overflow in adding offs, size was "negative"
+        jl_throw(jl_memory_exception);
+    v = jl_mmtk_gc_alloc_non_moving(ptls, allocsz, ty);
+
+    jl_set_typeof(v, ty);
+    maybe_record_alloc_to_profile(v, sz, (jl_datatype_t*)ty);
+    return v;
+}
 #endif // MMTK_GC
 
 /* Programming style note: When using jl_gc_alloc, do not JL_GC_PUSH it into a
@@ -525,6 +540,17 @@ JL_DLLEXPORT jl_value_t *jl_gc_alloc(jl_ptls_t ptls, size_t sz, void *ty);
 #  define jl_gc_alloc(ptls, sz, ty) jl_gc_alloc_(ptls, sz, ty)
 #endif
 
+JL_DLLEXPORT jl_value_t *jl_gc_alloc_non_moving(jl_ptls_t ptls, size_t sz, void *ty);
+
+#ifdef __GNUC__
+#  define jl_gc_alloc_non_moving(ptls, sz, ty)  \
+    (__builtin_constant_p(sz) ?      \
+     jl_gc_alloc__non_moving(ptls, sz, ty) :    \
+     (jl_gc_alloc_non_moving)(ptls, sz, ty))
+#else
+#  define jl_gc_alloc_non_moving(ptls, sz, ty) jl_gc_alloc__non_moving(ptls, sz, ty)
+#endif
+
 // jl_buff_tag must be an actual pointer here, so it cannot be confused for an actual type reference.
 // defined as uint64_t[3] so that we can get the right alignment of this and a "type tag" on it
 const extern uint64_t _jl_buff_tag[3];
@@ -534,7 +560,7 @@ JL_DLLEXPORT uintptr_t jl_get_buff_tag(void);
 typedef void jl_gc_tracked_buffer_t; // For the benefit of the static analyzer
 STATIC_INLINE jl_gc_tracked_buffer_t *jl_gc_alloc_buf(jl_ptls_t ptls, size_t sz)
 {
-    return jl_gc_alloc(ptls, sz, (void*)jl_buff_tag);
+    return jl_gc_alloc_non_moving(ptls, sz, (void*)jl_buff_tag);
 }
 
 STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT