Only reset unlog bits of objects in modbuf for nursery GCs or if in b…

…ootimage

This fixes a bug wherein we were erroneously resetting the unlog bits
for objects in the modbuf for full-heap GCs. If we reset the unlog bits
in a full-heap GC, we may end up setting the unlog bit for an object
that actually died which may cause issues for future allocations.

This PR also adds debug assertions checking that any object that has
been added to the modbuf is considered "live" by MMTk, or in other
words, it is a mature object.

src/plan/barriers.rs

@@ -231,6 +231,11 @@ impl<S: BarrierSemantics> Barrier<S::VM> for ObjectBarrier<S> {
         target: Option<ObjectReference>,
     ) {
         if self.log_object(src) {
+            debug_assert!(
+                src.is_live(),
+                "{} was logged but is not live",
+                src
+            );
             self.semantics
                 .object_reference_write_slow(src, slot, target);
         }

src/plan/generational/gc_work.rs

@@ -108,29 +108,53 @@ impl<E: ProcessEdgesWork> ProcessModBuf<E> {
 
 impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessModBuf<E> {
     fn do_work(&mut self, worker: &mut GCWorker<E::VM>, mmtk: &'static MMTK<E::VM>) {
-        // Flip the per-object unlogged bits to "unlogged" state.
-        for obj in &self.modbuf {
-            <E::VM as VMBinding>::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<E::VM, u8>(
-                *obj,
-                1,
-                None,
-                Ordering::SeqCst,
-            );
-        }
-        // scan modbuf only if the current GC is a nursery GC
+        // Process and scan modbuf only if the current GC is a nursery GC
         if mmtk
             .get_plan()
             .generational()
             .unwrap()
             .is_current_gc_nursery()
         {
+            // Flip the per-object unlogged bits to "unlogged" state.
+            for obj in &self.modbuf {
+                debug_assert!(
+                    (*obj).is_live(),
+                    "{} was logged but is not live",
+                    *obj
+                );
+
+                <E::VM as VMBinding>::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<E::VM, u8>(
+                    *obj,
+                    1,
+                    None,
+                    Ordering::SeqCst,
+                );
+            }
             // Scan objects in the modbuf and forward pointers
             let modbuf = std::mem::take(&mut self.modbuf);
             GCWork::do_work(
                 &mut ScanObjects::<E>::new(modbuf, false, WorkBucketStage::Closure),
                 worker,
                 mmtk,
             )
+        } else {
+            // Flip the per-object unlogged bits to "unlogged" state for objects inside the bootimage
+            #[cfg(feature = "vm_space")]
+            {
+                use crate::policy::space::Space;
+
+                for obj in &self.modbuf {
+                    if mmtk.get_plan().base().vm_space.in_space(*obj) {
+                        <E::VM as VMBinding>::VMObjectModel::GLOBAL_LOG_BIT_SPEC.store_atomic::<E::VM, u8>(
+                            *obj,
+                            1,
+                            None,
+                            Ordering::SeqCst,
+                        );
+                    }
+                }
+
+            }
         }
     }
 }