Allow setting object metadata for VM space objects. Expose VO bit under a feature.

Cargo.toml

@@ -183,6 +183,16 @@ bpftrace_workaround = []
 # users can disable such annotations by enabling this Cargo feature.
 no_mmap_annotation = []
 
+# Allow the binding to access Valid Object (VO) bit.
+# MMTk uses VO bit to identify a valid object. Thus VO bit is carefully managed by MMTk in cooperation with the binding.
+# So normally this feature is not needed, and its use should be discouraged.
+# However, in rare cases, a binding may want to directly access and manipulate VO bit. For example, a binding cannot cooperate
+# with MMTk to identify each object for setting the VO bit. This is usually due to the limitation of the VM.
+# In such cases, a binding may use this feature and manipulate the VO bit to their needs. The binding's manipulation on VO bit
+# may violate MMTk's semantics, and may result in undefined behaviors for VO bit related APIs. This feature should
+# only be used if you understand how VO bit works internally. Use at your own risk.
+vo_bit_access = []
+
 # Do not modify the following line - ci-common.sh matches it
 # -- Mutally exclusive features --
 # Only one feature from each group can be provided. Otherwise build will fail.

src/mmtk.rs

@@ -584,4 +584,16 @@ impl<VM: VMBinding> MMTK<VM> {
         });
         result_so_far
     }
+
+    /// Initialize object metadata for a VM space object.
+    /// Objects in the VM space are allocated/managed by the binding. This function provides a way for
+    /// the binding to set object metadata in MMTk for an object in the space.
+    #[cfg(feature = "vm_space")]
+    pub fn initialize_vm_space_object(&self, object: crate::util::ObjectReference) {
+        use crate::policy::sft::SFT;
+        self.get_plan()
+            .base()
+            .vm_space
+            .initialize_object_metadata(object, false)
+    }
 }

src/util/metadata/mod.rs

@@ -222,11 +222,11 @@ mod global;
 pub mod header_metadata;
 mod metadata_val_traits;
 pub mod side_metadata;
+pub mod vo_bit;
 pub use metadata_val_traits::*;
 
 pub(crate) mod log_bit;
 pub(crate) mod mark_bit;
 pub(crate) mod pin_bit;
-pub(crate) mod vo_bit;
 
 pub use global::*;

src/util/metadata/side_metadata/constants.rs

@@ -30,6 +30,7 @@ pub(crate) const GLOBAL_SIDE_METADATA_BASE_OFFSET: SideMetadataOffset =
     SideMetadataOffset::addr(GLOBAL_SIDE_METADATA_BASE_ADDRESS);
 
 /// Base address of VO bit, public to VM bindings which may need to use this.
+#[cfg(target_pointer_width = "64")]
 pub const VO_BIT_SIDE_METADATA_ADDR: Address =
     crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_ADDR;
 

src/util/metadata/vo_bit/mod.rs

@@ -54,26 +54,38 @@ use crate::util::ObjectReference;
 use crate::vm::object_model::ObjectModel;
 use crate::vm::VMBinding;
 
-/// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
-pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
-    crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+cfg_if::cfg_if! {
+    if #[cfg(feature = "vo_bit_access")] {
+        /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
+        /// This is only publicly available when the feature "vo_bit_access" is enabled.
+        /// Check the comments on "vo_bit_access" in `Cargo.toml` before use. Use at your own risk.
+        pub const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
+        crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+    } else {
+        /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata.
+        pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec =
+            crate::util::metadata::side_metadata::spec_defs::VO_BIT;
+    }
+}
 
+/// The base address for VO bit side metadata on 64 bits platforms.
+#[cfg(target_pointer_width = "64")]
 pub const VO_BIT_SIDE_METADATA_ADDR: Address = VO_BIT_SIDE_METADATA_SPEC.get_absolute_offset();
 
 /// Atomically set the VO bit for an object.
-pub fn set_vo_bit(object: ObjectReference) {
+pub(crate) fn set_vo_bit(object: ObjectReference) {
     debug_assert!(!is_vo_bit_set(object), "{:x}: VO bit already set", object);
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 1, Ordering::SeqCst);
 }
 
 /// Atomically unset the VO bit for an object.
-pub fn unset_vo_bit(object: ObjectReference) {
+pub(crate) fn unset_vo_bit(object: ObjectReference) {
     debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object);
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 0, Ordering::SeqCst);
 }
 
 /// Atomically unset the VO bit for an object, regardless whether the bit is set or not.
-pub fn unset_vo_bit_nocheck(object: ObjectReference) {
+pub(crate) fn unset_vo_bit_nocheck(object: ObjectReference) {
     VO_BIT_SIDE_METADATA_SPEC.store_atomic::<u8>(object.to_raw_address(), 0, Ordering::SeqCst);
 }
 
@@ -83,21 +95,21 @@ pub fn unset_vo_bit_nocheck(object: ObjectReference) {
 /// # Safety
 ///
 /// This is unsafe: check the comment on `side_metadata::store`
-pub unsafe fn unset_vo_bit_unsafe(object: ObjectReference) {
+pub(crate) unsafe fn unset_vo_bit_unsafe(object: ObjectReference) {
     debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object);
     VO_BIT_SIDE_METADATA_SPEC.store::<u8>(object.to_raw_address(), 0);
 }
 
 /// Check if the VO bit is set for an object.
-pub fn is_vo_bit_set(object: ObjectReference) -> bool {
+pub(crate) fn is_vo_bit_set(object: ObjectReference) -> bool {
     VO_BIT_SIDE_METADATA_SPEC.load_atomic::<u8>(object.to_raw_address(), Ordering::SeqCst) == 1
 }
 
 /// Check if an address can be turned directly into an object reference using the VO bit.
 /// If so, return `Some(object)`. Otherwise return `None`.
 ///
 /// The `address` must be word-aligned.
-pub fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
+pub(crate) fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
     is_vo_bit_set_inner::<true>(address)
 }
 
@@ -110,7 +122,7 @@ pub fn is_vo_bit_set_for_addr(address: Address) -> Option<ObjectReference> {
 /// # Safety
 ///
 /// This is unsafe: check the comment on `side_metadata::load`
-pub unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option<ObjectReference> {
+pub(crate) unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option<ObjectReference> {
     is_vo_bit_set_inner::<false>(address)
 }
 
@@ -135,7 +147,7 @@ fn is_vo_bit_set_inner<const ATOMIC: bool>(addr: Address) -> Option<ObjectRefere
 }
 
 /// Bulk zero the VO bit.
-pub fn bzero_vo_bit(start: Address, size: usize) {
+pub(crate) fn bzero_vo_bit(start: Address, size: usize) {
     VO_BIT_SIDE_METADATA_SPEC.bzero_metadata(start, size);
 }
 
@@ -145,7 +157,7 @@ pub fn bzero_vo_bit(start: Address, size: usize) {
 /// As an alternative, this function copies the mark bits metadata to VO bits.
 /// The caller needs to ensure the mark bits are set exactly wherever VO bits need to be set before
 /// calling this function.
-pub fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
+pub(crate) fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
     let mark_bit_spec = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC;
     debug_assert!(
         mark_bit_spec.is_on_side(),
@@ -158,19 +170,19 @@ pub fn bcopy_vo_bit_from_mark_bit<VM: VMBinding>(start: Address, size: usize) {
 use crate::util::constants::{LOG_BITS_IN_BYTE, LOG_BYTES_IN_ADDRESS};
 
 /// How many data memory bytes does 1 word in the VO bit side metadata represents?
-pub const VO_BIT_WORD_TO_REGION: usize = 1
+pub(crate) const VO_BIT_WORD_TO_REGION: usize = 1
     << (VO_BIT_SIDE_METADATA_SPEC.log_bytes_in_region
         + LOG_BITS_IN_BYTE as usize
         + LOG_BYTES_IN_ADDRESS as usize
         - VO_BIT_SIDE_METADATA_SPEC.log_num_of_bits);
 
 /// Bulk check if a VO bit word. Return true if there is any bit set in the word.
-pub fn get_raw_vo_bit_word(addr: Address) -> usize {
+pub(crate) fn get_raw_vo_bit_word(addr: Address) -> usize {
     unsafe { VO_BIT_SIDE_METADATA_SPEC.load_raw_word(addr) }
 }
 
 /// Find the base reference to the object from a potential internal pointer.
-pub fn find_object_from_internal_pointer<VM: VMBinding>(
+pub(crate) fn find_object_from_internal_pointer<VM: VMBinding>(
     start: Address,
     search_limit_bytes: usize,
 ) -> Option<ObjectReference> {
@@ -203,7 +215,7 @@ fn is_internal_ptr<VM: VMBinding>(obj: ObjectReference, internal_ptr: Address) -
 }
 
 /// Check if the address could be an internal pointer based on where VO bit is set.
-pub fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
+pub(crate) fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
     vo_addr: Address,
     internal_ptr: Address,
 ) -> Option<ObjectReference> {
@@ -219,6 +231,6 @@ pub fn is_internal_ptr_from_vo_bit<VM: VMBinding>(
 ///
 /// # Safety
 /// The caller needs to make sure that no one is modifying VO bit.
-pub unsafe fn is_vo_addr(addr: Address) -> bool {
+pub(crate) unsafe fn is_vo_addr(addr: Address) -> bool {
     VO_BIT_SIDE_METADATA_SPEC.load::<u8>(addr) != 0
 }