experiment: precise heap tags

rts/motoko-rts-tests/src/continuation_table.rs

@@ -6,7 +6,7 @@ use motoko_rts::continuation_table::{
     continuation_count, recall_continuation, remember_continuation,
 };
 use motoko_rts::memory::alloc_blob;
-use motoko_rts::types::{size_of, Array, Blob, Bytes, Value, Words};
+use motoko_rts::types::{size_of, Array, Blob, Bytes, Value, Words, TAG_BLOB_B};
 
 pub unsafe fn test() {
     println!("Testing continuation table ...");
@@ -22,7 +22,7 @@ pub unsafe fn test() {
             + N as u32 * size_of::<Blob>().to_bytes().as_u32(),
     ));
 
-    let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, Bytes(0)));
+    let pointers: [Value; N] = from_fn(|_| alloc_blob(&mut heap, TAG_BLOB_B, Bytes(0)));
 
     let mut references: [u32; N] = [0; N];
     for i in 0..N {

rts/motoko-rts-tests/src/gc.rs

@@ -271,7 +271,7 @@ fn check_dynamic_heap(
 
             let is_forwarded = forward != make_pointer(address as u32);
 
-            if incremental && tag == TAG_BLOB {
+            if incremental && tag == TAG_BLOB_B {
                 assert!(!is_forwarded);
                 // in-heap mark stack blobs
                 let length = read_word(heap, offset);
@@ -285,9 +285,14 @@ fn check_dynamic_heap(
                     .as_usize();
             } else {
                 if incremental {
-                    assert!(tag == TAG_ARRAY || tag >= TAG_ARRAY_SLICE_MIN);
+                    assert!(
+                        tag == TAG_ARRAY_I
+                            || tag == TAG_ARRAY_M
+                            || tag == TAG_ARRAY_T
+                            || tag >= TAG_ARRAY_SLICE_MIN
+                    );
                 } else {
-                    assert_eq!(tag, TAG_ARRAY);
+                    assert!(tag == TAG_ARRAY_I || tag == TAG_ARRAY_M || tag == TAG_ARRAY_T)
                 }
 
                 if is_forwarded {
@@ -431,7 +436,7 @@ fn check_continuation_table(mut offset: usize, continuation_table: &[ObjectIdx],
     let incremental = cfg!(feature = "incremental_gc");
 
     let table_addr = heap.as_ptr() as usize + offset;
-    assert_eq!(read_word(heap, offset), TAG_ARRAY);
+    assert_eq!(read_word(heap, offset), TAG_ARRAY_M);
     offset += WORD_SIZE;
 
     if incremental {

rts/motoko-rts-tests/src/gc/compacting/mark_stack.rs

@@ -33,18 +33,26 @@ fn test_push_pop() {
         .unwrap();
 }
 
-static TAGS: [Tag; 14] = [
+static TAGS: [Tag; 22] = [
     TAG_OBJECT,
     TAG_OBJ_IND,
-    TAG_ARRAY,
-    TAG_BITS64,
+    TAG_ARRAY_I,
+    TAG_ARRAY_M,
+    TAG_ARRAY_T,
+    TAG_BITS64_U,
+    TAG_BITS64_S,
+    TAG_BITS64_F,
     TAG_MUTBOX,
     TAG_CLOSURE,
     TAG_SOME,
     TAG_VARIANT,
-    TAG_BLOB,
+    TAG_BLOB_B,
+    TAG_BLOB_T,
+    TAG_BLOB_P,
     TAG_FWD_PTR,
-    TAG_BITS32,
+    TAG_BITS32_U,
+    TAG_BITS32_S,
+    TAG_BITS32_F,
     TAG_BIGINT,
     TAG_CONCAT,
     TAG_NULL,

rts/motoko-rts-tests/src/gc/heap.rs

@@ -442,7 +442,7 @@ fn create_dynamic_heap(
 
             // Store object header
             let address = u32::try_from(heap_start + heap_offset).unwrap();
-            write_word(dynamic_heap, heap_offset, TAG_ARRAY);
+            write_word(dynamic_heap, heap_offset, TAG_ARRAY_M);
             heap_offset += WORD_SIZE;
 
             if incremental {
@@ -497,7 +497,7 @@ fn create_dynamic_heap(
         let mut heap_offset = continuation_table_offset;
 
         let continuation_table_address = u32::try_from(heap_start + heap_offset).unwrap();
-        write_word(dynamic_heap, heap_offset, TAG_ARRAY);
+        write_word(dynamic_heap, heap_offset, TAG_ARRAY_M); // TODO: TBR
         heap_offset += WORD_SIZE;
 
         if incremental {
@@ -570,7 +570,7 @@ fn create_static_heap(
     // root.
     let array_addr = u32::try_from(heap.as_ptr() as usize).unwrap();
     let mut offset = 0;
-    write_word(heap, offset, TAG_ARRAY);
+    write_word(heap, offset, TAG_ARRAY_M); // TODO: TBR
     offset += WORD_SIZE;
 
     if incremental {

rts/motoko-rts-tests/src/gc/incremental/array_slicing.rs

@@ -1,33 +1,37 @@
 use motoko_rts::{
     gc::incremental::array_slicing::slice_array,
     memory::alloc_array,
-    types::{Words, TAG_ARRAY, TAG_ARRAY_SLICE_MIN},
+    types::{Tag, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_SLICE_MIN, TAG_ARRAY_T},
 };
 
 use crate::memory::TestMemory;
 
 pub unsafe fn test() {
     println!("  Testing array slicing...");
 
-    let mut mem = TestMemory::new(Words(1024 * 1024));
-    // multiple of slice increment
-    test_array_slicing(&mut mem, 4096);
-    // odd remainder of slice increment
-    test_array_slicing(&mut mem, 3999);
-    // small array
-    test_array_slicing(&mut mem, 10);
-    // empty array
-    test_array_slicing(&mut mem, 0);
+    let tags = vec![TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T];
+
+    for tag in tags.into_iter() {
+        let mut mem = TestMemory::new(Words(1024 * 1024));
+        // multiple of slice increment
+        test_array_slicing(&mut mem, tag, 4096);
+        // odd remainder of slice increment
+        test_array_slicing(&mut mem, tag, 3999);
+        // small array
+        test_array_slicing(&mut mem, tag, 10);
+        // empty array
+        test_array_slicing(&mut mem, tag, 0);
+    }
 }
 
-unsafe fn test_array_slicing(mem: &mut TestMemory, array_length: u32) {
-    let array = alloc_array(mem, array_length).as_array();
+unsafe fn test_array_slicing(mem: &mut TestMemory, tag: Tag, array_length: u32) {
+    let array = alloc_array(mem, tag, array_length).as_array();
     let mut last_offset = 0;
     loop {
         let new_offset = slice_array(array);
         assert!(new_offset > last_offset || array.len() == 0 && new_offset == 0);
         last_offset = new_offset;
-        if (*array).header.tag == TAG_ARRAY {
+        if (*array).header.tag == tag {
             break;
         }
         assert!((*array).header.tag >= TAG_ARRAY_SLICE_MIN);

rts/motoko-rts-tests/src/gc/incremental/partitioned_heap.rs

@@ -13,7 +13,10 @@ use motoko_rts::{
         time::BoundedTime,
     },
     memory::{alloc_array, alloc_blob, Memory},
-    types::{Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY, TAG_BLOB},
+    types::{
+        Array, Blob, Bytes, Obj, Tag, Value, Words, TAG_ARRAY_I, TAG_ARRAY_M, TAG_ARRAY_T,
+        TAG_BLOB_B, TAG_BLOB_P, TAG_BLOB_T,
+    },
 };
 
 use crate::{gc::utils::WORD_SIZE, memory::TestMemory};
@@ -308,7 +311,7 @@ unsafe fn iterate_large_partition(
     let mut time = BoundedTime::new(0);
     while iterator.has_object() {
         let object = iterator.current_object();
-        assert_eq!(object.tag(), TAG_BLOB);
+        assert_eq!(object.tag(), TAG_BLOB_B);
         let size = block_size(object as *const Tag);
         detected_sizes.push(size);
         time.tick();
@@ -396,7 +399,7 @@ impl PartitionedTestHeap {
 
     pub fn allocate_array(&mut self, elements: &[Value]) -> Value {
         unsafe {
-            let array = alloc_array(self, elements.len() as u32);
+            let array = alloc_array(self, TAG_ARRAY_M, elements.len() as u32);
             for index in 0..elements.len() {
                 let raw_array = array.as_array();
                 raw_array.set_scalar(index as u32, elements[index]);
@@ -406,16 +409,18 @@ impl PartitionedTestHeap {
     }
 
     pub fn allocate_blob(&mut self, size: usize) -> Value {
-        unsafe { alloc_blob(self, Bytes(size as u32)) }
+        unsafe { alloc_blob(self, TAG_BLOB_B, Bytes(size as u32)) }
     }
 }
 
 unsafe fn block_size(block: *const Tag) -> usize {
     match *block {
-        TAG_ARRAY => {
+        TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T => {
             size_of::<Array>() + (block as *const Array).len() as usize * WORD_SIZE as usize
         }
-        TAG_BLOB => size_of::<Blob>() + (block as *const Blob).len().as_usize(),
+        TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P => {
+            size_of::<Blob>() + (block as *const Blob).len().as_usize()
+        }
         _ => unimplemented!(),
     }
 }

rts/motoko-rts-tests/src/principal_id.rs

@@ -1,7 +1,7 @@
 use crate::memory::TestMemory;
 
 use motoko_rts::principal_id::{blob_of_principal, principal_of_blob};
-use motoko_rts::text::{text_compare, text_of_ptr_size, text_of_str};
+use motoko_rts::text::{blob_compare, text_compare, text_of_ptr_size, text_of_str};
 use motoko_rts::types::{Bytes, Words};
 
 pub unsafe fn test() {
@@ -36,11 +36,11 @@ pub unsafe fn test() {
 
     let text = text_of_str(&mut heap, "aaaaa-aa");
     let principal = blob_of_principal(&mut heap, text);
-    assert_eq!(text_compare(principal, text_of_str(&mut heap, ""),), 0);
+    assert_eq!(blob_compare(principal, text_of_str(&mut heap, ""),), 0);
 
     let text = text_of_str(&mut heap, "bfozs-kwa73-7nadi");
     assert_eq!(
-        text_compare(
+        blob_compare(
             blob_of_principal(&mut heap, text),
             text_of_ptr_size(&mut heap, b"\xC0\xFE\xFE\xD0\x0D".as_ptr(), Bytes(5))
         ),

rts/motoko-rts-tests/src/text.rs

@@ -8,7 +8,7 @@ use motoko_rts::text::{
     text_singleton, text_size,
 };
 use motoko_rts::text_iter::{text_iter, text_iter_done, text_iter_next};
-use motoko_rts::types::{Bytes, Value, Words, TAG_BLOB};
+use motoko_rts::types::{Bytes, Value, Words, TAG_BLOB_T};
 
 use std::convert::TryFrom;
 
@@ -68,7 +68,7 @@ pub unsafe fn test() {
     for i in 0..8 {
         let str = &STR[0..i + 1];
         let text = text_of_str(&mut mem, str);
-        assert_eq!(text.tag(), TAG_BLOB);
+        assert_eq!(text.tag(), TAG_BLOB_T);
         let iter = TextIter::from_text(&mut mem, text);
         assert_eq!(iter.collect::<String>(), str);
     }

rts/motoko-rts/src/allocator.rs

@@ -3,7 +3,7 @@
 use alloc::alloc::{GlobalAlloc, Layout};
 //use core::ptr::null_mut;
 use crate::memory::{alloc_blob, ic};
-use crate::types::Bytes;
+use crate::types::{Bytes, TAG_BLOB_B};
 
 pub struct EphemeralAllocator;
 
@@ -24,7 +24,8 @@ unsafe impl GlobalAlloc for EphemeralAllocator {
         let min_align = (align + word_size - 1) / word_size * word_size;
         let blob_size = size + min_align - word_size;
         let blob =
-            alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, Bytes(blob_size as u32)).as_blob_mut();
+            alloc_blob::<ic::IcMemory>(&mut ic::IcMemory, TAG_BLOB_B, Bytes(blob_size as u32))
+                .as_blob_mut();
         let payload_address = blob.payload_addr() as usize;
         let aligned_address = (payload_address + min_align - 1) / min_align * min_align;
 

rts/motoko-rts/src/blob_iter.rs

@@ -1,6 +1,6 @@
 use crate::{
     barriers::allocation_barrier,
-    types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY},
+    types::{size_of, Array, Bytes, Value, Words, TAG_ARRAY_T},
 };
 
 use motoko_rts_macros::ic_mem_fn;
@@ -16,7 +16,7 @@ unsafe fn blob_iter<M: crate::memory::Memory>(mem: &mut M, blob: Value) -> Value
 
     // NB. cannot use as_array() here as we didn't write the header yet
     let iter_array = iter_ptr.get_ptr() as *mut Array;
-    (*iter_array).header.tag = TAG_ARRAY;
+    (*iter_array).header.tag = TAG_ARRAY_T;
     (*iter_array).header.init_forward(iter_ptr);
     (*iter_array).len = 2;
 

rts/motoko-rts/src/continuation_table.rs

@@ -25,7 +25,7 @@
 use crate::barriers::allocation_barrier;
 use crate::memory::{alloc_array, Memory};
 use crate::rts_trap_with;
-use crate::types::Value;
+use crate::types::{Value, TAG_ARRAY_M};
 
 use motoko_rts_macros::ic_mem_fn;
 
@@ -42,7 +42,7 @@ static mut N_CONTINUATIONS: u32 = 0;
 static mut FREE_SLOT: u32 = 0;
 
 unsafe fn create_continuation_table<M: Memory>(mem: &mut M) {
-    TABLE = alloc_array(mem, INITIAL_SIZE);
+    TABLE = alloc_array(mem, TAG_ARRAY_M, INITIAL_SIZE);
     FREE_SLOT = 0;
     N_CONTINUATIONS = 0;
 
@@ -61,7 +61,7 @@ unsafe fn double_continuation_table<M: Memory>(mem: &mut M) {
 
     let new_size = old_size * 2;
 
-    TABLE = alloc_array(mem, new_size);
+    TABLE = alloc_array(mem, TAG_ARRAY_M, new_size);
     let new_array = TABLE.as_array();
 
     for i in 0..old_size {

rts/motoko-rts/src/debug.rs

@@ -165,7 +165,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
             let obj_ind = obj as *const ObjInd;
             let _ = write!(buf, "<ObjInd field={:#x}>", (*obj_ind).field.get_raw());
         }
-        TAG_ARRAY | TAG_ARRAY_SLICE_MIN.. => {
+        TAG_ARRAY_I | TAG_ARRAY_M | TAG_ARRAY_T | TAG_ARRAY_SLICE_MIN.. => {
             let array = obj as *mut Array;
             let _ = write!(buf, "<Array len={:#x}", (*array).len);
 
@@ -179,7 +179,7 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
                 let _ = write!(buf, ">");
             }
         }
-        TAG_BITS64 => {
+        TAG_BITS64_U | TAG_BITS64_S | TAG_BITS64_F => {
             let bits64 = obj as *const Bits64;
             let _ = write!(buf, "<Bits64 {:#x}>", (*bits64).bits());
         }
@@ -204,15 +204,15 @@ pub(crate) unsafe fn print_boxed_object(buf: &mut WriteBuf, p: usize) {
                 (*variant).field.get_raw()
             );
         }
-        TAG_BLOB => {
+        TAG_BLOB_B | TAG_BLOB_T | TAG_BLOB_P => {
             let blob = obj.as_blob();
             let _ = write!(buf, "<Blob len={:#x}>", blob.len().as_u32());
         }
         TAG_FWD_PTR => {
             let ind = obj as *const FwdPtr;
             let _ = write!(buf, "<Forwarding to {:#x}>", (*ind).fwd.get_raw());
         }
-        TAG_BITS32 => {
+        TAG_BITS32_U | TAG_BITS32_S | TAG_BITS32_F => {
             let bits32 = obj as *const Bits32;
             let _ = write!(buf, "<Bits32 {:#x}>", (*bits32).bits);
         }

rts/motoko-rts/src/gc/generational.rs

@@ -296,15 +296,16 @@ impl<'a, M: Memory> GenerationalGC<'a, M> {
             |gc, slice_start, array| {
                 const SLICE_INCREMENT: u32 = 255;
                 debug_assert!(SLICE_INCREMENT >= TAG_ARRAY_SLICE_MIN);
+                let base_tag = array.base_tag();
                 if array.len() - slice_start > SLICE_INCREMENT {
                     let new_start = slice_start + SLICE_INCREMENT;
                     // Remember to visit the array suffix later, store the next visit offset in the tag.
-                    (*array).header.tag = new_start;
+                    array.set_slice_start(base_tag, new_start);
                     push_mark_stack(gc.heap.mem, array as usize);
                     new_start
                 } else {
                     // No further visits of this array. Restore the tag.
-                    (*array).header.tag = TAG_ARRAY;
+                    array.restore_tag(base_tag); // restore original tag
                     array.len()
                 }
             },

rts/motoko-rts/src/gc/generational/mark_stack.rs

@@ -2,7 +2,7 @@
 //! Simplified to only store object pointers without tags.
 
 use crate::memory::{alloc_blob, Memory};
-use crate::types::{Blob, Words};
+use crate::types::{Blob, Words, TAG_BLOB_B};
 
 use core::ptr::null_mut;
 
@@ -27,7 +27,7 @@ pub unsafe fn alloc_mark_stack<M: Memory>(mem: &mut M) {
 
     // Allocating an actual object here to not break dump_heap
     // No post allocation barrier as this RTS-internal blob will be collected by the GC.
-    STACK_BLOB_PTR = alloc_blob(mem, INIT_STACK_SIZE.to_bytes()).get_ptr() as *mut Blob;
+    STACK_BLOB_PTR = alloc_blob(mem, TAG_BLOB_B, INIT_STACK_SIZE.to_bytes()).get_ptr() as *mut Blob;
     STACK_BASE = STACK_BLOB_PTR.payload_addr() as *mut usize;
     STACK_PTR = STACK_BASE;
     STACK_TOP = STACK_BASE.add(INIT_STACK_SIZE.as_usize());