diff --git a/Cargo.toml b/Cargo.toml index 94e44a445c..525247fb50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,6 +183,16 @@ bpftrace_workaround = [] # users can disable such annotations by enabling this Cargo feature. no_mmap_annotation = [] +# Allow the binding to access Valid Object (VO) bit. +# MMTk uses VO bit to identify a valid object. Thus VO bit is carefully managed by MMTk in cooperation with the binding. +# So normally this feature is not needed, and its use should be discouraged. +# However, in rare cases, a binding may want to directly access and manipulate VO bit. For example, a binding cannot cooperate +# with MMTk to identify each object for setting the VO bit. This is usually due to the limitation of the VM. +# In such cases, a binding may use this feature and manipulate the VO bit to their needs. The binding's manipulation on VO bit +# may violate MMTk's semantics, and may result in undefined behaviors for VO bit related APIs. This feature should +# only be used if you understand how VO bit works internally. Use at your own risk. +vo_bit_access = [] + # Do not modify the following line - ci-common.sh matches it # -- Mutally exclusive features -- # Only one feature from each group can be provided. Otherwise build will fail. diff --git a/src/mmtk.rs b/src/mmtk.rs index 24aa57d6c7..59ee97132a 100644 --- a/src/mmtk.rs +++ b/src/mmtk.rs @@ -584,4 +584,16 @@ impl MMTK { }); result_so_far } + + /// Initialize object metadata for a VM space object. + /// Objects in the VM space are allocated/managed by the binding. This function provides a way for + /// the binding to set object metadata in MMTk for an object in the space. + #[cfg(feature = "vm_space")] + pub fn initialize_vm_space_object(&self, object: crate::util::ObjectReference) { + use crate::policy::sft::SFT; + self.get_plan() + .base() + .vm_space + .initialize_object_metadata(object, false) + } } diff --git a/src/util/metadata/mod.rs b/src/util/metadata/mod.rs index 4b64f38bcc..6f386bbe04 100644 --- a/src/util/metadata/mod.rs +++ b/src/util/metadata/mod.rs @@ -222,11 +222,11 @@ mod global; pub mod header_metadata; mod metadata_val_traits; pub mod side_metadata; +pub mod vo_bit; pub use metadata_val_traits::*; pub(crate) mod log_bit; pub(crate) mod mark_bit; pub(crate) mod pin_bit; -pub(crate) mod vo_bit; pub use global::*; diff --git a/src/util/metadata/side_metadata/constants.rs b/src/util/metadata/side_metadata/constants.rs index 6a8bbdcca5..f04c94d9d1 100644 --- a/src/util/metadata/side_metadata/constants.rs +++ b/src/util/metadata/side_metadata/constants.rs @@ -30,6 +30,7 @@ pub(crate) const GLOBAL_SIDE_METADATA_BASE_OFFSET: SideMetadataOffset = SideMetadataOffset::addr(GLOBAL_SIDE_METADATA_BASE_ADDRESS); /// Base address of VO bit, public to VM bindings which may need to use this. +#[cfg(target_pointer_width = "64")] pub const VO_BIT_SIDE_METADATA_ADDR: Address = crate::util::metadata::vo_bit::VO_BIT_SIDE_METADATA_ADDR; diff --git a/src/util/metadata/vo_bit/mod.rs b/src/util/metadata/vo_bit/mod.rs index dc2b90b916..b5794e4075 100644 --- a/src/util/metadata/vo_bit/mod.rs +++ b/src/util/metadata/vo_bit/mod.rs @@ -54,26 +54,38 @@ use crate::util::ObjectReference; use crate::vm::object_model::ObjectModel; use crate::vm::VMBinding; -/// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata. -pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec = - crate::util::metadata::side_metadata::spec_defs::VO_BIT; +cfg_if::cfg_if! { + if #[cfg(feature = "vo_bit_access")] { + /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata. + /// This is only publicly available when the feature "vo_bit_access" is enabled. + /// Check the comments on "vo_bit_access" in `Cargo.toml` before use. Use at your own risk. + pub const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec = + crate::util::metadata::side_metadata::spec_defs::VO_BIT; + } else { + /// A VO bit is required per min-object-size aligned address, rather than per object, and can only exist as side metadata. + pub(crate) const VO_BIT_SIDE_METADATA_SPEC: SideMetadataSpec = + crate::util::metadata::side_metadata::spec_defs::VO_BIT; + } +} +/// The base address for VO bit side metadata on 64 bits platforms. +#[cfg(target_pointer_width = "64")] pub const VO_BIT_SIDE_METADATA_ADDR: Address = VO_BIT_SIDE_METADATA_SPEC.get_absolute_offset(); /// Atomically set the VO bit for an object. -pub fn set_vo_bit(object: ObjectReference) { +pub(crate) fn set_vo_bit(object: ObjectReference) { debug_assert!(!is_vo_bit_set(object), "{:x}: VO bit already set", object); VO_BIT_SIDE_METADATA_SPEC.store_atomic::(object.to_raw_address(), 1, Ordering::SeqCst); } /// Atomically unset the VO bit for an object. -pub fn unset_vo_bit(object: ObjectReference) { +pub(crate) fn unset_vo_bit(object: ObjectReference) { debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object); VO_BIT_SIDE_METADATA_SPEC.store_atomic::(object.to_raw_address(), 0, Ordering::SeqCst); } /// Atomically unset the VO bit for an object, regardless whether the bit is set or not. -pub fn unset_vo_bit_nocheck(object: ObjectReference) { +pub(crate) fn unset_vo_bit_nocheck(object: ObjectReference) { VO_BIT_SIDE_METADATA_SPEC.store_atomic::(object.to_raw_address(), 0, Ordering::SeqCst); } @@ -83,13 +95,13 @@ pub fn unset_vo_bit_nocheck(object: ObjectReference) { /// # Safety /// /// This is unsafe: check the comment on `side_metadata::store` -pub unsafe fn unset_vo_bit_unsafe(object: ObjectReference) { +pub(crate) unsafe fn unset_vo_bit_unsafe(object: ObjectReference) { debug_assert!(is_vo_bit_set(object), "{:x}: VO bit not set", object); VO_BIT_SIDE_METADATA_SPEC.store::(object.to_raw_address(), 0); } /// Check if the VO bit is set for an object. -pub fn is_vo_bit_set(object: ObjectReference) -> bool { +pub(crate) fn is_vo_bit_set(object: ObjectReference) -> bool { VO_BIT_SIDE_METADATA_SPEC.load_atomic::(object.to_raw_address(), Ordering::SeqCst) == 1 } @@ -97,7 +109,7 @@ pub fn is_vo_bit_set(object: ObjectReference) -> bool { /// If so, return `Some(object)`. Otherwise return `None`. /// /// The `address` must be word-aligned. -pub fn is_vo_bit_set_for_addr(address: Address) -> Option { +pub(crate) fn is_vo_bit_set_for_addr(address: Address) -> Option { is_vo_bit_set_inner::(address) } @@ -110,7 +122,7 @@ pub fn is_vo_bit_set_for_addr(address: Address) -> Option { /// # Safety /// /// This is unsafe: check the comment on `side_metadata::load` -pub unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option { +pub(crate) unsafe fn is_vo_bit_set_unsafe(address: Address) -> Option { is_vo_bit_set_inner::(address) } @@ -135,7 +147,7 @@ fn is_vo_bit_set_inner(addr: Address) -> Option(start: Address, size: usize) { +pub(crate) fn bcopy_vo_bit_from_mark_bit(start: Address, size: usize) { let mark_bit_spec = VM::VMObjectModel::LOCAL_MARK_BIT_SPEC; debug_assert!( mark_bit_spec.is_on_side(), @@ -158,19 +170,19 @@ pub fn bcopy_vo_bit_from_mark_bit(start: Address, size: usize) { use crate::util::constants::{LOG_BITS_IN_BYTE, LOG_BYTES_IN_ADDRESS}; /// How many data memory bytes does 1 word in the VO bit side metadata represents? -pub const VO_BIT_WORD_TO_REGION: usize = 1 +pub(crate) const VO_BIT_WORD_TO_REGION: usize = 1 << (VO_BIT_SIDE_METADATA_SPEC.log_bytes_in_region + LOG_BITS_IN_BYTE as usize + LOG_BYTES_IN_ADDRESS as usize - VO_BIT_SIDE_METADATA_SPEC.log_num_of_bits); /// Bulk check if a VO bit word. Return true if there is any bit set in the word. -pub fn get_raw_vo_bit_word(addr: Address) -> usize { +pub(crate) fn get_raw_vo_bit_word(addr: Address) -> usize { unsafe { VO_BIT_SIDE_METADATA_SPEC.load_raw_word(addr) } } /// Find the base reference to the object from a potential internal pointer. -pub fn find_object_from_internal_pointer( +pub(crate) fn find_object_from_internal_pointer( start: Address, search_limit_bytes: usize, ) -> Option { @@ -203,7 +215,7 @@ fn is_internal_ptr(obj: ObjectReference, internal_ptr: Address) - } /// Check if the address could be an internal pointer based on where VO bit is set. -pub fn is_internal_ptr_from_vo_bit( +pub(crate) fn is_internal_ptr_from_vo_bit( vo_addr: Address, internal_ptr: Address, ) -> Option { @@ -219,6 +231,6 @@ pub fn is_internal_ptr_from_vo_bit( /// /// # Safety /// The caller needs to make sure that no one is modifying VO bit. -pub unsafe fn is_vo_addr(addr: Address) -> bool { +pub(crate) unsafe fn is_vo_addr(addr: Address) -> bool { VO_BIT_SIDE_METADATA_SPEC.load::(addr) != 0 }