Allow setting object metadata for VM space objects. Expose VO bit und…

…er a feature. (#1248)

This PR changes a few things for vo bit:
1. Add a function to set object metadata for an object in the VM space:
`MMTK::initialize_vm_space_object`.
2. Add a feature `vo_bit_access` to expose VO bit and a binding may use
it at its own risk.
3. Mark VO bit side metadata base address only avilable for 64 bits.

The second is needed for Julia. The Julia binding uses MMTk immortal
allocation or VM space for a region of memory, and pop the regions with
boot image objects with no clear way to identify each object. The easist
workaround is to bulk set VO bit for the region. The problem from this
is that MMTk cannot identify valid objects in those regions. However,
Julia binding only uses VO bit for pinning objects. Objects in the
immortal space or the VM space will not be moved so failing to pinning
objects in those regions is benign. Currently the Julia binding
duplicates a bunch of side metadata code to bulk set VO bit only using
the VO bit side metadata base address. See
mmtk/mmtk-julia#200

Cargo.toml

@@ -183,6 +183,16 @@ bpftrace_workaround = []
 # users can disable such annotations by enabling this Cargo feature.
 no_mmap_annotation = []
 
+# Allow the binding to access Valid Object (VO) bit.
+# MMTk uses VO bit to identify a valid object. Thus VO bit is carefully managed by MMTk in cooperation with the binding.
+# So normally this feature is not needed, and its use should be discouraged.
+# However, in rare cases, a binding may want to directly access and manipulate VO bit. For example, a binding cannot cooperate
+# with MMTk to identify each object for setting the VO bit. This is usually due to the limitation of the VM.
+# In such cases, a binding may use this feature and manipulate the VO bit to their needs. The binding's manipulation on VO bit
+# may violate MMTk's semantics, and may result in undefined behaviors for VO bit related APIs. This feature should
+# only be used if you understand how VO bit works internally. Use at your own risk.
+vo_bit_access = []
+
 # Do not modify the following line - ci-common.sh matches it
 # -- Mutally exclusive features --
 # Only one feature from each group can be provided. Otherwise build will fail.

src/mmtk.rs

@@ -584,4 +584,16 @@ impl<VM: VMBinding> MMTK<VM> {
         });
         result_so_far
     }
+
+    /// Initialize object metadata for a VM space object.
+    /// Objects in the VM space are allocated/managed by the binding. This function provides a way for
+    /// the binding to set object metadata in MMTk for an object in the space.
+    #[cfg(feature = "vm_space")]
+    pub fn initialize_vm_space_object(&self, object: crate::util::ObjectReference) {
+        use crate::policy::sft::SFT;
+        self.get_plan()
+            .base()
+            .vm_space
+            .initialize_object_metadata(object, false)
+    }
 }

src/util/metadata/mod.rs

@@ -222,11 +222,11 @@ mod global;
 pub mod header_metadata;
 mod metadata_val_traits;
 pub mod side_metadata;
+pub mod vo_bit;
 pub use metadata_val_traits::*;
 
 pub(crate) mod log_bit;
 pub(crate) mod mark_bit;
 pub(crate) mod pin_bit;
-pub(crate) mod vo_bit;
 
 pub use global::*;

src/util/metadata/side_metadata/constants.rs

@@ -30,6 +30,7 @@ pub(crate) const GLOBAL_SIDE_METADATA_BASE_OFFSET: SideMetadataOffset =
     SideMetadataOffset::addr(GLOBAL_SIDE_METADATA_BASE_ADDRESS);
 
 /// Base address of VO bit, public to VM bindings which may need to use this.
+#[cfg(target_pointer_width = "64")]
 pub const VO_BIT_SIDE_METADATA_ADDR: Address =
     crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_ADDR;
 

src/util/metadata/vo_bit/mod.rs

@@ -54,26 +54,38 @@ use crate::util::ObjectReference;
 use crate::vm::object_model::ObjectModel;
 use crate::vm::VMBinding;
 
-/// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
-pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
-    crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+cfg_if::cfg_if! {
+    if #[cfg(feature = "vo_bit_access")] {
+        /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
+        /// This is only publicly available when the feature "vo_bit_access" is enabled.
+        /// Check the comments on "vo_bit_access" in `Cargo.toml` before use. Use at your own risk.
+        pub const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
+        crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+    } else {
+        /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
+        pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
+            crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+    }
+}
 
+/// The base address for VO bit side metadata on 64 bits platforms.
+#[cfg(target_pointer_width = "64")]
 pub const VO_BIT_SIDE_METADATA_ADDR: Address = VO_BIT_SIDE_METADATA_SPEC.get_absolute_offset();
 
 /// Atomically set the VO bit for an object.
-pub fn set_vo_bit(object: ObjectReference) {
+pub(crate) fn set_vo_bit(object: ObjectReference) {
     debug_assert!(!is_vo_bit_set(object), "{:x}: VO bit already set", object);
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 1, Ordering::SeqCst);
 }
 
 /// Atomically unset the VO bit for an object.
-pub fn unset_vo_bit(object: ObjectReference) {
+pub(crate) fn unset_vo_bit(object: ObjectReference) {
     debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object);
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 0, Ordering::SeqCst);
 }
 
 /// Atomically unset the VO bit for an object, regardless whether the bit is set or not.
-pub fn unset_vo_bit_nocheck(object: ObjectReference) {
+pub(crate) fn unset_vo_bit_nocheck(object: ObjectReference) {
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 0, Ordering::SeqCst);
 }
 
@@ -83,21 +95,21 @@ pub fn unset_vo_bit_nocheck(object: ObjectReference) {
 /// # Safety
 ///
 /// This is unsafe: check the comment on `side_metadata::store`
-pub unsafe fn unset_vo_bit_unsafe(object: ObjectReference) {
+pub(crate) unsafe fn unset_vo_bit_unsafe(object: ObjectReference) {
     debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object);
     VO_BIT_SIDE_METADATA_SPEC.store::<u8>(object.to_raw_address(), 0);
 }
 
 /// Check if the VO bit is set for an object.
-pub fn is_vo_bit_set(object: ObjectReference) -> bool {
+pub(crate) fn is_vo_bit_set(object: ObjectReference) -> bool {
     VO_BIT_SIDE_METADATA_SPEC.load_atomic::<u8>(object.to_raw_address(), Ordering::SeqCst) == 1
 }
 
 /// Check if an address can be turned directly into an object reference using the VO bit.
 /// If so, return `Some(object)`. Otherwise return `None`.
 ///
 /// The `address` must be word-aligned.
-pub fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
+pub(crate) fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
     is_vo_bit_set_inner::<true>(address)
 }
 
@@ -110,7 +122,7 @@ pub fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
 /// # Safety
 ///
 /// This is unsafe: check the comment on `side_metadata::load`
-pub unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option<ObjectReference> {
+pub(crate) unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option<ObjectReference> {
     is_vo_bit_set_inner::<false>(address)
 }
 
@@ -135,7 +147,7 @@ fn is_vo_bit_set_inner<const ATOMIC: bool>(addr: Address) -> Option<ObjectRefere
 }
 
 /// Bulk zero the VO bit.
-pub fn bzero_vo_bit(start: Address, size: usize) {
+pub(crate) fn bzero_vo_bit(start: Address, size: usize) {
     VO_BIT_SIDE_METADATA_SPEC.bzero_metadata(start, size);
 }
 
@@ -145,7 +157,7 @@ pub fn bzero_vo_bit(start: Address, size: usize) {
 /// As an alternative, this function copies the mark bits metadata to VO bits.
 /// The caller needs to ensure the mark bits are set exactly wherever VO bits need to be set before
 /// calling this function.
-pub fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
+pub(crate) fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
     let mark_bit_spec = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC;
     debug_assert!(
         mark_bit_spec.is_on_side(),
@@ -158,19 +170,19 @@ pub fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
 use crate::util::constants::{LOG_BITS_IN_BYTE, LOG_BYTES_IN_ADDRESS};
 
 /// How many data memory bytes does 1 word in the VO bit side metadata represents?
-pub const VO_BIT_WORD_TO_REGION: usize = 1
+pub(crate) const VO_BIT_WORD_TO_REGION: usize = 1
     << (VO_BIT_SIDE_METADATA_SPEC.log_bytes_in_region
         + LOG_BITS_IN_BYTE as usize
         + LOG_BYTES_IN_ADDRESS as usize
         - VO_BIT_SIDE_METADATA_SPEC.log_num_of_bits);
 
 /// Bulk check if a VO bit word. Return true if there is any bit set in the word.
-pub fn get_raw_vo_bit_word(addr: Address) -> usize {
+pub(crate) fn get_raw_vo_bit_word(addr: Address) -> usize {
     unsafe { VO_BIT_SIDE_METADATA_SPEC.load_raw_word(addr) }
 }
 
 /// Find the base reference to the object from a potential internal pointer.
-pub fn find_object_from_internal_pointer<VM: VMBinding>(
+pub(crate) fn find_object_from_internal_pointer<VM: VMBinding>(
     start: Address,
     search_limit_bytes: usize,
 ) -> Option<ObjectReference> {
@@ -203,7 +215,7 @@ fn is_internal_ptr<VM: VMBinding>(obj: ObjectReference, internal_ptr: Address) -
 }
 
 /// Check if the address could be an internal pointer based on where VO bit is set.
-pub fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
+pub(crate) fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
     vo_addr: Address,
     internal_ptr: Address,
 ) -> Option<ObjectReference> {
@@ -219,6 +231,6 @@ pub fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
 ///
 /// # Safety
 /// The caller needs to make sure that no one is modifying VO bit.
-pub unsafe fn is_vo_addr(addr: Address) -> bool {
+pub(crate) unsafe fn is_vo_addr(addr: Address) -> bool {
     VO_BIT_SIDE_METADATA_SPEC.load::<u8>(addr) != 0
 }