From 61d20e2dcd5b4743ef04a8118eb807bcd6f6e2e2 Mon Sep 17 00:00:00 2001 From: Eduardo Souza Date: Mon, 11 Sep 2023 10:02:31 +1000 Subject: [PATCH] This PR enables transitively pinning (TP) objects from particular roots for Immix/StickyImmix (#897) - A new bucket `ImmovableClosure` has been created to trace and transitively pin roots, i.e., roots in which no object in its transitive closure is allowed to move. - `RootsWorkFactory` has two new functions `create_process_tp_edge_roots_work` and `create_process_tp_node_roots_work` to create work to process these roots. - `GCWorkContext` expects an `TPProcessEdges` type, which performs the transitively pinning trace (and is unimplemented for unsupported plans). - `create_process_node_roots_work` creates work to process roots in the `NodeRootsTrace` bucket, which is executed after `TPClosure`, but any work derived from it is put into the regular `Closure` bucket, meaning that the binding shouldn't need to pin root nodes. - For sticky immix, currently we only support non-moving nursery collections (`sticky_immix_non_moving_nursery`), but the solution here is to use the `TPClosure` on the modified buffer constructed from the write barrier (since during nursery collection we do not know if these objects should be transitively pinned or not). --------- Co-authored-by: Yi Lin Co-authored-by: Kunshan Wang --- .github/scripts/replace-mmtk-dep.py | 13 +- Cargo.toml | 6 + .../tutorial/code/mygc_semispace/gc_work.rs | 24 +- src/plan/generational/copying/gc_work.rs | 4 +- src/plan/generational/gc_work.rs | 21 +- src/plan/generational/immix/gc_work.rs | 3 + src/plan/immix/gc_work.rs | 2 + src/plan/immix/global.rs | 4 +- src/plan/markcompact/gc_work.rs | 16 +- src/plan/markcompact/global.rs | 2 +- src/plan/marksweep/gc_work.rs | 1 + src/plan/pageprotect/gc_work.rs | 1 + src/plan/semispace/gc_work.rs | 3 +- src/plan/sticky/immix/gc_work.rs | 3 + src/plan/tracing.rs | 8 +- src/policy/copyspace.rs | 5 + src/policy/gc_work.rs | 1 + src/policy/immix/immixspace.rs | 13 +- src/policy/immix/mod.rs | 3 +- src/policy/markcompactspace.rs | 6 +- src/scheduler/gc_work.rs | 350 +++++++++++++----- src/scheduler/scheduler.rs | 8 +- src/scheduler/work.rs | 8 +- src/scheduler/work_bucket.rs | 5 + src/util/finalizable_processor.rs | 6 +- src/util/reference_processor.rs | 9 +- src/util/sanity/sanity_checker.rs | 23 +- src/vm/scanning.rs | 19 +- tests/test_roots_work_factory.rs | 6 +- 29 files changed, 415 insertions(+), 158 deletions(-) diff --git a/.github/scripts/replace-mmtk-dep.py b/.github/scripts/replace-mmtk-dep.py index da811db5dd..3eadb5c577 100644 --- a/.github/scripts/replace-mmtk-dep.py +++ b/.github/scripts/replace-mmtk-dep.py @@ -19,10 +19,15 @@ mmtk_node = toml_data["dependencies"]["mmtk"] -print("Deleting dependencies.mmtk.git") -if "git" in mmtk_node: - del mmtk_node["git"] - +# These keys may specify the locations of the dependency. Remove them. +for key in ["git", "branch", "version", "registry"]: + if key in mmtk_node: + print("Deleting dependencies.mmtk.{}".format(key)) + del mmtk_node[key] + else: + print("Key dependencies.mmtk.{} does not exist. Ignored.".format(key)) + +# Use mmtk-core from the specified local directory. mmtk_repo_path = os.path.realpath(args.mmtk_core_path) print("Setting dependencies.mmtk.path to {}".format(mmtk_repo_path)) mmtk_node["path"] = mmtk_repo_path diff --git a/Cargo.toml b/Cargo.toml index 04454fa85a..ca172f6a81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,6 +105,12 @@ object_pinning = [] # Disable any object copying in Immix. This makes Immix a non-moving policy. immix_non_moving = [] +# Disable any object copying in nursery GC for Sticky Immix while allowing other kinds of copying. +# `immix_non_moving` disables all kinds of copying in Immix, so this feature is not needed +# if `immix_non_moving` is in use. +sticky_immix_non_moving_nursery = [] + + # Reduce block size for ImmixSpace. This mitigates fragmentation when defrag is disabled. immix_smaller_block = [] # Zero the unmarked lines after a GC cycle in immix. This helps debug untraced objects. diff --git a/docs/userguide/src/tutorial/code/mygc_semispace/gc_work.rs b/docs/userguide/src/tutorial/code/mygc_semispace/gc_work.rs index f19c1e25b0..29be4f6184 100644 --- a/docs/userguide/src/tutorial/code/mygc_semispace/gc_work.rs +++ b/docs/userguide/src/tutorial/code/mygc_semispace/gc_work.rs @@ -1,6 +1,6 @@ // ANCHOR: imports use super::global::MyGC; -use crate::scheduler::gc_work::*; +use crate::scheduler::{gc_work::*, WorkBucketStage}; use crate::vm::VMBinding; use std::ops::{Deref, DerefMut}; // ANCHOR_END: imports @@ -11,24 +11,26 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = SFTProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR_END: workcontext_sft // ANCHOR: workcontext_plan -use crate::scheduler::gc_work::PlanProcessEdges; use crate::policy::gc_work::DEFAULT_TRACE; +use crate::scheduler::gc_work::PlanProcessEdges; pub struct MyGCWorkContext2(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for MyGCWorkContext2 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR_END: workcontext_plan -use crate::util::ObjectReference; +use crate::policy::space::Space; use crate::util::copy::CopySemantics; +use crate::util::ObjectReference; use crate::MMTK; -use crate::policy::space::Space; // ANCHOR: mygc_process_edges pub struct MyGCProcessEdges { @@ -38,12 +40,17 @@ pub struct MyGCProcessEdges { // ANCHOR_END: mygc_process_edges // ANCHOR: mygc_process_edges_impl -impl ProcessEdgesWork for MyGCProcessEdges { +impl ProcessEdgesWork for MyGCProcessEdges { type VM = VM; type ScanObjectsWorkType = ScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref::>().unwrap(); Self { base, plan } } @@ -74,7 +81,7 @@ impl ProcessEdgesWork for MyGCProcessEdges { } fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, self.bucket) } } // ANCHOR_END: mygc_process_edges_impl @@ -100,5 +107,6 @@ impl crate::scheduler::GCWorkContext for MyGCWorkContext3 { type VM = VM; type PlanType = MyGC; type ProcessEdgesWorkType = MyGCProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } // ANCHOR: workcontext_mygc diff --git a/src/plan/generational/copying/gc_work.rs b/src/plan/generational/copying/gc_work.rs index e94b43d71b..226ddab715 100644 --- a/src/plan/generational/copying/gc_work.rs +++ b/src/plan/generational/copying/gc_work.rs @@ -3,13 +3,14 @@ use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::vm::*; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; pub struct GenCopyNurseryGCWorkContext(std::marker::PhantomData); impl crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkContext { type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } pub struct GenCopyGCWorkContext(std::marker::PhantomData); @@ -17,4 +18,5 @@ impl crate::scheduler::GCWorkContext for GenCopyGCWorkContext type VM = VM; type PlanType = GenCopy; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/generational/gc_work.rs b/src/plan/generational/gc_work.rs index 88abf09c48..a5c38e84ea 100644 --- a/src/plan/generational/gc_work.rs +++ b/src/plan/generational/gc_work.rs @@ -1,7 +1,7 @@ use atomic::Ordering; use crate::plan::PlanTraceObject; -use crate::scheduler::{gc_work::*, GCWork, GCWorker}; +use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage}; use crate::util::ObjectReference; use crate::vm::edge_shape::{Edge, MemorySlice}; use crate::vm::*; @@ -25,8 +25,13 @@ impl + PlanTraceObject> ProcessEdg type VM = VM; type ScanObjectsWorkType = PlanScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref().unwrap(); Self { plan, base } } @@ -51,7 +56,7 @@ impl + PlanTraceObject> ProcessEdg nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - PlanScanObjects::new(self.plan, nodes, false, roots) + PlanScanObjects::new(self.plan, nodes, false, roots, self.bucket) } } @@ -111,7 +116,7 @@ impl GCWork for ProcessModBuf { // Scan objects in the modbuf and forward pointers let modbuf = std::mem::take(&mut self.modbuf); GCWork::do_work( - &mut ScanObjects::::new(modbuf, false, false), + &mut ScanObjects::::new(modbuf, false, false, WorkBucketStage::Closure), worker, mmtk, ) @@ -154,7 +159,11 @@ impl GCWork for ProcessRegionModBuf { } } // Forward entries - GCWork::do_work(&mut E::new(edges, false, mmtk), worker, mmtk) + GCWork::do_work( + &mut E::new(edges, false, mmtk, WorkBucketStage::Closure), + worker, + mmtk, + ) } } } diff --git a/src/plan/generational/immix/gc_work.rs b/src/plan/generational/immix/gc_work.rs index ea6b220935..14f46a0958 100644 --- a/src/plan/generational/immix/gc_work.rs +++ b/src/plan/generational/immix/gc_work.rs @@ -2,6 +2,7 @@ use super::global::GenImmix; use crate::plan::generational::gc_work::GenNurseryProcessEdges; use crate::policy::gc_work::TraceKind; use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::UnsupportedProcessEdges; use crate::vm::VMBinding; pub struct GenImmixNurseryGCWorkContext(std::marker::PhantomData); @@ -9,6 +10,7 @@ impl crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkCon type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } pub(super) struct GenImmixMatureGCWorkContext( @@ -20,4 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = GenImmix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/immix/gc_work.rs b/src/plan/immix/gc_work.rs index 0696086fc1..e3200f8811 100644 --- a/src/plan/immix/gc_work.rs +++ b/src/plan/immix/gc_work.rs @@ -1,5 +1,6 @@ use super::global::Immix; use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; use crate::vm::VMBinding; @@ -12,4 +13,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = Immix; type ProcessEdgesWorkType = PlanProcessEdges, KIND>; + type TPProcessEdges = PlanProcessEdges, TRACE_KIND_TRANSITIVE_PIN>; } diff --git a/src/plan/immix/global.rs b/src/plan/immix/global.rs index b4ed586087..23e2b0402e 100644 --- a/src/plan/immix/global.rs +++ b/src/plan/immix/global.rs @@ -157,8 +157,8 @@ impl Immix { /// to schedule a full heap collection. A plan must call set_collection_kind and set_gc_status before this method. pub(crate) fn schedule_immix_full_heap_collection< PlanType: Plan, - FastContext: 'static + GCWorkContext, - DefragContext: 'static + GCWorkContext, + FastContext: GCWorkContext, + DefragContext: GCWorkContext, >( plan: &'static DefragContext::PlanType, immix_space: &ImmixSpace, diff --git a/src/plan/markcompact/gc_work.rs b/src/plan/markcompact/gc_work.rs index 990cadc84d..5ff937e6a4 100644 --- a/src/plan/markcompact/gc_work.rs +++ b/src/plan/markcompact/gc_work.rs @@ -57,12 +57,13 @@ impl GCWork for UpdateReferences { .get_and_clear_worker_live_bytes(); for mutator in VM::VMActivePlan::mutators() { - mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] - .add(ScanMutatorRoots::>(mutator)); + mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots].add(ScanMutatorRoots::< + MarkCompactForwardingGCWorkContext, + >(mutator)); } mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots] - .add(ScanVMSpecificRoots::>::new()); + .add(ScanVMSpecificRoots::>::new()); } } @@ -102,4 +103,13 @@ impl crate::scheduler::GCWorkContext for MarkCompactGCWorkContext type VM = VM; type PlanType = MarkCompact; type ProcessEdgesWorkType = MarkingProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; +} + +pub struct MarkCompactForwardingGCWorkContext(std::marker::PhantomData); +impl crate::scheduler::GCWorkContext for MarkCompactForwardingGCWorkContext { + type VM = VM; + type PlanType = MarkCompact; + type ProcessEdgesWorkType = ForwardingProcessEdges; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/markcompact/global.rs b/src/plan/markcompact/global.rs index 964276ba58..32da425ea3 100644 --- a/src/plan/markcompact/global.rs +++ b/src/plan/markcompact/global.rs @@ -92,7 +92,7 @@ impl Plan for MarkCompact { // Stop & scan mutators (mutator scanning can happen before STW) scheduler.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::>::new()); + .add(StopMutators::>::new()); // Prepare global/collectors/mutators scheduler.work_buckets[WorkBucketStage::Prepare] diff --git a/src/plan/marksweep/gc_work.rs b/src/plan/marksweep/gc_work.rs index 90ed731dd6..604e355759 100644 --- a/src/plan/marksweep/gc_work.rs +++ b/src/plan/marksweep/gc_work.rs @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for MSGCWorkContext { type VM = VM; type PlanType = MarkSweep; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type TPProcessEdges = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/pageprotect/gc_work.rs b/src/plan/pageprotect/gc_work.rs index c962eae190..4e68c4afb2 100644 --- a/src/plan/pageprotect/gc_work.rs +++ b/src/plan/pageprotect/gc_work.rs @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for PPGCWorkContext { type VM = VM; type PlanType = PageProtect; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type TPProcessEdges = PlanProcessEdges, DEFAULT_TRACE>; } diff --git a/src/plan/semispace/gc_work.rs b/src/plan/semispace/gc_work.rs index 9f02c73eea..1d4e633c36 100644 --- a/src/plan/semispace/gc_work.rs +++ b/src/plan/semispace/gc_work.rs @@ -1,6 +1,6 @@ use super::global::SemiSpace; use crate::policy::gc_work::DEFAULT_TRACE; -use crate::scheduler::gc_work::PlanProcessEdges; +use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges}; use crate::vm::VMBinding; pub struct SSGCWorkContext(std::marker::PhantomData); @@ -8,4 +8,5 @@ impl crate::scheduler::GCWorkContext for SSGCWorkContext { type VM = VM; type PlanType = SemiSpace; type ProcessEdgesWorkType = PlanProcessEdges, DEFAULT_TRACE>; + type TPProcessEdges = UnsupportedProcessEdges; } diff --git a/src/plan/sticky/immix/gc_work.rs b/src/plan/sticky/immix/gc_work.rs index 0afdc44ee0..3c7b6f973a 100644 --- a/src/plan/sticky/immix/gc_work.rs +++ b/src/plan/sticky/immix/gc_work.rs @@ -1,4 +1,5 @@ use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::scheduler::gc_work::PlanProcessEdges; use crate::{plan::generational::gc_work::GenNurseryProcessEdges, vm::VMBinding}; @@ -9,6 +10,7 @@ impl crate::scheduler::GCWorkContext for StickyImmixNurseryGCWork type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = GenNurseryProcessEdges; + type TPProcessEdges = GenNurseryProcessEdges; } pub struct StickyImmixMatureGCWorkContext( @@ -20,4 +22,5 @@ impl crate::scheduler::GCWorkContext type VM = VM; type PlanType = StickyImmix; type ProcessEdgesWorkType = PlanProcessEdges; + type TPProcessEdges = PlanProcessEdges; } diff --git a/src/plan/tracing.rs b/src/plan/tracing.rs index e2011f9b38..dc4eaac6df 100644 --- a/src/plan/tracing.rs +++ b/src/plan/tracing.rs @@ -75,13 +75,15 @@ impl ObjectQueue for VectorQueue { pub struct ObjectsClosure<'a, E: ProcessEdgesWork> { buffer: VectorQueue>, pub(crate) worker: &'a mut GCWorker, + bucket: WorkBucketStage, } impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> { - pub fn new(worker: &'a mut GCWorker) -> Self { + pub fn new(worker: &'a mut GCWorker, bucket: WorkBucketStage) -> Self { Self { buffer: VectorQueue::new(), worker, + bucket, } } @@ -89,8 +91,8 @@ impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> { let buf = self.buffer.take(); if !buf.is_empty() { self.worker.add_work( - WorkBucketStage::Closure, - E::new(buf, false, self.worker.mmtk), + self.bucket, + E::new(buf, false, self.worker.mmtk, self.bucket), ); } } diff --git a/src/policy/copyspace.rs b/src/policy/copyspace.rs index 4d45beb4c0..1dcb2ce813 100644 --- a/src/policy/copyspace.rs +++ b/src/policy/copyspace.rs @@ -1,5 +1,6 @@ use crate::plan::{ObjectQueue, VectorObjectQueue}; use crate::policy::copy_context::PolicyCopyContext; +use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::space::{CommonSpace, Space}; @@ -126,6 +127,10 @@ impl crate::policy::gc_work::PolicyTraceObject for CopySpace< copy: Option, worker: &mut GCWorker, ) -> ObjectReference { + debug_assert!( + KIND != TRACE_KIND_TRANSITIVE_PIN, + "Copyspace does not support transitive pin trace." + ); self.trace_object(queue, object, copy, worker) } diff --git a/src/policy/gc_work.rs b/src/policy/gc_work.rs index 15dabd7f96..389618b1e0 100644 --- a/src/policy/gc_work.rs +++ b/src/policy/gc_work.rs @@ -3,6 +3,7 @@ pub(crate) type TraceKind = u8; pub const DEFAULT_TRACE: u8 = u8::MAX; +pub const TRACE_KIND_TRANSITIVE_PIN: u8 = DEFAULT_TRACE - 1; use crate::plan::ObjectQueue; use crate::scheduler::GCWorker; diff --git a/src/policy/immix/immixspace.rs b/src/policy/immix/immixspace.rs index 01354d920d..fb578fefe8 100644 --- a/src/policy/immix/immixspace.rs +++ b/src/policy/immix/immixspace.rs @@ -1,7 +1,7 @@ use super::line::*; use super::{block::*, defrag::Defrag}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_TRANSITIVE_PIN}; use crate::policy::sft::GCWorkerMutRef; use crate::policy::sft::SFT; use crate::policy::sft_map::SFTMap; @@ -192,7 +192,9 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace copy: Option, worker: &mut GCWorker, ) -> ObjectReference { - if KIND == TRACE_KIND_DEFRAG { + if KIND == TRACE_KIND_TRANSITIVE_PIN { + self.trace_object_without_moving(queue, object) + } else if KIND == TRACE_KIND_DEFRAG { if Block::containing::(object).is_defrag_source() { debug_assert!(self.in_defrag()); debug_assert!( @@ -227,7 +229,7 @@ impl crate::policy::gc_work::PolicyTraceObject for ImmixSpace fn may_move_objects() -> bool { if KIND == TRACE_KIND_DEFRAG { true - } else if KIND == TRACE_KIND_FAST { + } else if KIND == TRACE_KIND_FAST || KIND == TRACE_KIND_TRANSITIVE_PIN { false } else { unreachable!() @@ -619,11 +621,6 @@ impl ImmixSpace { } else if self.is_marked(object) { // We won the forwarding race but the object is already marked so we clear the // forwarding status and return the unmoved object - debug_assert!( - nursery_collection || self.defrag.space_exhausted() || self.is_pinned(object), - "Forwarded object is the same as original object {} even though it should have been copied", - object, - ); ForwardingWord::clear_forwarding_bits::(object); object } else { diff --git a/src/policy/immix/mod.rs b/src/policy/immix/mod.rs index fb9f0846f9..32c94261e9 100644 --- a/src/policy/immix/mod.rs +++ b/src/policy/immix/mod.rs @@ -25,7 +25,8 @@ pub const STRESS_DEFRAG: bool = false; pub const DEFRAG_EVERY_BLOCK: bool = false; /// If Immix is used as a nursery space, do we prefer copy? -pub const PREFER_COPY_ON_NURSERY_GC: bool = !cfg!(feature = "immix_non_moving"); // copy nursery objects if we are allowed to move. +pub const PREFER_COPY_ON_NURSERY_GC: bool = + !cfg!(feature = "immix_non_moving") && !cfg!(feature = "sticky_immix_non_moving_nursery"); // copy nursery objects if we are allowed to move. /// In some cases/settings, Immix may never move objects. /// Currently we only have two cases where we move objects: 1. defrag, 2. nursery copy. diff --git a/src/policy/markcompactspace.rs b/src/policy/markcompactspace.rs index d44d7a78ce..693218b492 100644 --- a/src/policy/markcompactspace.rs +++ b/src/policy/markcompactspace.rs @@ -3,7 +3,7 @@ use std::ops::Range; use super::sft::SFT; use super::space::{CommonSpace, Space}; use crate::plan::VectorObjectQueue; -use crate::policy::gc_work::TraceKind; +use crate::policy::gc_work::{TraceKind, TRACE_KIND_TRANSITIVE_PIN}; use crate::policy::sft::GCWorkerMutRef; use crate::scheduler::GCWorker; use crate::util::alloc::allocator::align_allocation_no_fill; @@ -130,6 +130,10 @@ impl crate::policy::gc_work::PolicyTraceObject for MarkCompac _copy: Option, _worker: &mut GCWorker, ) -> ObjectReference { + debug_assert!( + KIND != TRACE_KIND_TRANSITIVE_PIN, + "MarkCompact does not support transitive pin trace." + ); if KIND == TRACE_KIND_MARK { self.trace_mark_object(queue, object) } else if KIND == TRACE_KIND_FORWARD { diff --git a/src/scheduler/gc_work.rs b/src/scheduler/gc_work.rs index 47d6d70761..ebf45d5c77 100644 --- a/src/scheduler/gc_work.rs +++ b/src/scheduler/gc_work.rs @@ -44,7 +44,7 @@ impl Prepare { } } -impl GCWork for Prepare { +impl GCWork for Prepare { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("Prepare Global"); // We assume this is the only running work packet that accesses plan at the point of execution @@ -183,27 +183,27 @@ impl GCWork for ReleaseCollector { /// /// TODO: Smaller work granularity #[derive(Default)] -pub struct StopMutators(PhantomData); +pub struct StopMutators(PhantomData); -impl StopMutators { +impl StopMutators { pub fn new() -> Self { Self(PhantomData) } } -impl GCWork for StopMutators { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for StopMutators { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("stop_all_mutators start"); mmtk.get_plan().base().prepare_for_stack_scanning(); - ::VMCollection::stop_all_mutators(worker.tls, |mutator| { + ::VMCollection::stop_all_mutators(worker.tls, |mutator| { // TODO: The stack scanning work won't start immediately, as the `Prepare` bucket is not opened yet (the bucket is opened in notify_mutators_paused). // Should we push to Unconstrained instead? mmtk.scheduler.work_buckets[WorkBucketStage::Prepare] - .add(ScanMutatorRoots::(mutator)); + .add(ScanMutatorRoots::(mutator)); }); trace!("stop_all_mutators end"); mmtk.scheduler.notify_mutators_paused(mmtk); - mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanVMSpecificRoots::::new()); + mmtk.scheduler.work_buckets[WorkBucketStage::Prepare].add(ScanVMSpecificRoots::::new()); } } @@ -330,7 +330,7 @@ impl ObjectTracerContext for ProcessEdgesWorkTracerC let mmtk = worker.mmtk; // Prepare the underlying ProcessEdgesWork - let mut process_edges_work = E::new(vec![], false, mmtk); + let mut process_edges_work = E::new(vec![], false, mmtk, self.stage); // FIXME: This line allows us to omit the borrowing lifetime of worker. // We should refactor ProcessEdgesWork so that it uses `worker` locally, not as a member. process_edges_work.set_worker(worker); @@ -446,15 +446,19 @@ impl GCWork for VMPostForwarding { } } -pub struct ScanMutatorRoots(pub &'static mut Mutator); +pub struct ScanMutatorRoots(pub &'static mut Mutator); -impl GCWork for ScanMutatorRoots { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ScanMutatorRoots { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanMutatorRoots for mutator {:?}", self.0.get_tls()); let base = mmtk.get_plan().base(); - let mutators = ::VMActivePlan::number_of_mutators(); - let factory = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - ::VMScanning::scan_roots_in_mutator_thread( + let mutators = ::VMActivePlan::number_of_mutators(); + let factory = ProcessEdgesWorkRootsWorkFactory::< + C::VM, + C::ProcessEdgesWorkType, + C::TPProcessEdges, + >::new(mmtk); + ::VMScanning::scan_roots_in_mutator_thread( worker.tls, unsafe { &mut *(self.0 as *mut _) }, factory, @@ -462,7 +466,7 @@ impl GCWork for ScanMutatorRoots { self.0.flush(); if mmtk.get_plan().base().inform_stack_scanned(mutators) { - ::VMScanning::notify_initial_thread_scan_complete( + ::VMScanning::notify_initial_thread_scan_complete( false, worker.tls, ); base.set_gc_status(GcStatus::GcProper); @@ -471,19 +475,23 @@ impl GCWork for ScanMutatorRoots { } #[derive(Default)] -pub struct ScanVMSpecificRoots(PhantomData); +pub struct ScanVMSpecificRoots(PhantomData); -impl ScanVMSpecificRoots { +impl ScanVMSpecificRoots { pub fn new() -> Self { Self(PhantomData) } } -impl GCWork for ScanVMSpecificRoots { - fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { +impl GCWork for ScanVMSpecificRoots { + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("ScanStaticRoots"); - let factory = ProcessEdgesWorkRootsWorkFactory::::new(mmtk); - ::VMScanning::scan_vm_specific_roots(worker.tls, factory); + let factory = ProcessEdgesWorkRootsWorkFactory::< + C::VM, + C::ProcessEdgesWorkType, + C::TPProcessEdges, + >::new(mmtk); + ::VMScanning::scan_vm_specific_roots(worker.tls, factory); } } @@ -495,6 +503,7 @@ pub struct ProcessEdgesBase { // Because a copying gc will dereference this pointer at least once for every object copy. worker: *mut GCWorker, pub roots: bool, + pub bucket: WorkBucketStage, } unsafe impl Send for ProcessEdgesBase {} @@ -502,7 +511,12 @@ unsafe impl Send for ProcessEdgesBase {} impl ProcessEdgesBase { // Requires an MMTk reference. Each plan-specific type that uses ProcessEdgesBase can get a static plan reference // at creation. This avoids overhead for dynamic dispatch or downcasting plan for each object traced. - pub fn new(edges: Vec, roots: bool, mmtk: &'static MMTK) -> Self { + pub fn new( + edges: Vec, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { #[cfg(feature = "extreme_assertions")] if crate::util::edge_logger::should_check_duplicate_edges(mmtk.get_plan()) { for edge in &edges { @@ -516,6 +530,7 @@ impl ProcessEdgesBase { mmtk, worker: std::ptr::null_mut(), roots, + bucket, } } pub fn set_worker(&mut self, worker: &mut GCWorker) { @@ -567,7 +582,12 @@ pub trait ProcessEdgesWork: const OVERWRITE_REFERENCE: bool = true; const SCAN_OBJECTS_IMMEDIATELY: bool = true; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self; + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self; /// Trace an MMTk object. The implementation should forward this call to the policy-specific /// `trace_object()` methods, depending on which space this object is in. @@ -596,14 +616,15 @@ pub trait ProcessEdgesWork: // Executing these work packets now can remarkably reduce the global synchronization time. self.worker().do_work(work_packet); } else { - self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(work_packet); + debug_assert!(self.bucket != WorkBucketStage::Unconstrained); + self.mmtk.scheduler.work_buckets[self.bucket].add(work_packet); } } /// Create an object-scanning work packet to be used for this ProcessEdgesWork. /// /// `roots` indicates if we are creating a packet for root scanning. It is only true when this - /// method is called to handle `RootsWorkFactory::create_process_node_roots_work`. + /// method is called to handle `RootsWorkFactory::create_process_pinning_roots_work`. fn create_scan_work( &self, nodes: Vec, @@ -665,8 +686,13 @@ impl ProcessEdgesWork for SFTProcessEdges { type VM = VM; type ScanObjectsWorkType = ScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); Self { base } } @@ -686,40 +712,67 @@ impl ProcessEdgesWork for SFTProcessEdges { } fn create_scan_work(&self, nodes: Vec, roots: bool) -> ScanObjects { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, self.bucket) } } - -struct ProcessEdgesWorkRootsWorkFactory { - mmtk: &'static MMTK, +struct ProcessEdgesWorkRootsWorkFactory< + VM: VMBinding, + E: ProcessEdgesWork, + I: ProcessEdgesWork, +> { + mmtk: &'static MMTK, + phantom: PhantomData<(E, I)>, } -impl Clone for ProcessEdgesWorkRootsWorkFactory { +impl, I: ProcessEdgesWork> Clone + for ProcessEdgesWorkRootsWorkFactory +{ fn clone(&self) -> Self { - Self { mmtk: self.mmtk } + Self { + mmtk: self.mmtk, + phantom: PhantomData, + } } } -impl RootsWorkFactory> for ProcessEdgesWorkRootsWorkFactory { +impl, I: ProcessEdgesWork> + RootsWorkFactory> for ProcessEdgesWorkRootsWorkFactory +{ fn create_process_edge_roots_work(&mut self, edges: Vec>) { crate::memory_manager::add_work_packet( self.mmtk, WorkBucketStage::Closure, - E::new(edges, true, self.mmtk), + E::new(edges, true, self.mmtk, WorkBucketStage::Closure), + ); + } + + fn create_process_pinning_roots_work(&mut self, nodes: Vec) { + // Will process roots within the PinningRootsTrace bucket + // And put work in the Closure bucket + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::PinningRootsTrace, + ProcessRootNode::::new(nodes, WorkBucketStage::Closure), ); } - fn create_process_node_roots_work(&mut self, nodes: Vec) { - // We want to use E::create_scan_work. - let process_edges_work = E::new(vec![], true, self.mmtk); - let work = process_edges_work.create_scan_work(nodes, true); - crate::memory_manager::add_work_packet(self.mmtk, WorkBucketStage::Closure, work); + fn create_process_tpinning_roots_work(&mut self, nodes: Vec) { + crate::memory_manager::add_work_packet( + self.mmtk, + WorkBucketStage::TPinningClosure, + ProcessRootNode::::new(nodes, WorkBucketStage::TPinningClosure), + ); } } -impl ProcessEdgesWorkRootsWorkFactory { - fn new(mmtk: &'static MMTK) -> Self { - Self { mmtk } +impl, I: ProcessEdgesWork> + ProcessEdgesWorkRootsWorkFactory +{ + fn new(mmtk: &'static MMTK) -> Self { + Self { + mmtk, + phantom: PhantomData, + } } } @@ -752,61 +805,25 @@ pub trait ScanObjectsWork: GCWork + Sized { /// the objects in this packet. fn make_another(&self, buffer: Vec) -> Self; + fn get_bucket(&self) -> WorkBucketStage; + /// The common code for ScanObjects and PlanScanObjects. fn do_work_common( &self, buffer: &[ObjectReference], worker: &mut GCWorker<::VM>, - mmtk: &'static MMTK<::VM>, + _mmtk: &'static MMTK<::VM>, ) { let tls = worker.tls; + debug_assert!(!self.roots()); - #[cfg(feature = "sanity")] - { - if self.roots() && !mmtk.get_plan().is_in_sanity() { - mmtk.sanity_checker - .lock() - .unwrap() - .add_root_nodes(buffer.to_vec()); - } - } - - // If this is a root packet, the objects in this packet will have not been traced, yet. - // - // This step conceptually traces the edges from root slots to the objects they point to. - // However, VMs that deliver root objects instead of root edges are incapable of updating - // root slots. Like processing an edge, we call `trace_object` on those objects, and - // assert the GC doesn't move those objects because we cannot store back to the slots. - // - // If this is a root packet, the `scanned_root_objects` variable will hold those root - // objects which are traced for the first time. - let scanned_root_objects = self.roots().then(|| { - // We create an instance of E to use its `trace_object` method and its object queue. - let mut process_edges_work = Self::E::new(vec![], false, mmtk); - process_edges_work.set_worker(worker); - - for object in buffer.iter().copied() { - let new_object = process_edges_work.trace_object(object); - debug_assert_eq!( - object, new_object, - "Object moved while tracing root unmovable root object: {} -> {}", - object, new_object - ); - } - - // This contains root objects that are visited the first time. - // It is sufficient to only scan these objects. - process_edges_work.nodes.take() - }); - - // If it is a root packet, scan the nodes that are first scanned; - // otherwise, scan the nodes in the buffer. - let objects_to_scan = scanned_root_objects.as_deref().unwrap_or(buffer); + // Scan the nodes in the buffer. + let objects_to_scan = buffer; // Then scan those objects for edges. let mut scan_later = vec![]; { - let mut closure = ObjectsClosure::::new(worker); + let mut closure = ObjectsClosure::::new(worker, self.get_bucket()); for object in objects_to_scan.iter().copied() { // For any object we need to scan, we count its liv bytes #[cfg(feature = "count_live_bytes_in_gc")] @@ -835,7 +852,7 @@ pub trait ScanObjectsWork: GCWork + Sized { // If any object does not support edge-enqueuing, we process them now. if !scan_later.is_empty() { let object_tracer_context = ProcessEdgesWorkTracerContext:: { - stage: WorkBucketStage::Closure, + stage: self.get_bucket(), phantom_data: PhantomData, }; @@ -868,15 +885,22 @@ pub struct ScanObjects { concurrent: bool, roots: bool, phantom: PhantomData, + bucket: WorkBucketStage, } impl ScanObjects { - pub fn new(buffer: Vec, concurrent: bool, roots: bool) -> Self { + pub fn new( + buffer: Vec, + concurrent: bool, + roots: bool, + bucket: WorkBucketStage, + ) -> Self { Self { buffer, concurrent, roots, phantom: PhantomData, + bucket, } } } @@ -888,12 +912,16 @@ impl> ScanObjectsWork for ScanOb self.roots } + fn get_bucket(&self) -> WorkBucketStage { + self.bucket + } + fn post_scan_object(&self, _object: ObjectReference) { // Do nothing. } fn make_another(&self, buffer: Vec) -> Self { - Self::new(buffer, self.concurrent, false) + Self::new(buffer, self.concurrent, false, self.bucket) } } @@ -927,8 +955,13 @@ impl + Plan, const KIND: TraceKin type VM = VM; type ScanObjectsWorkType = PlanScanObjects; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { - let base = ProcessEdgesBase::new(edges, roots, mmtk); + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { + let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket); let plan = base.plan().downcast_ref::

().unwrap(); Self { plan, base } } @@ -938,7 +971,7 @@ impl + Plan, const KIND: TraceKin nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - PlanScanObjects::::new(self.plan, nodes, false, roots) + PlanScanObjects::::new(self.plan, nodes, false, roots, self.bucket) } fn trace_object(&mut self, object: ObjectReference) -> ObjectReference { @@ -987,6 +1020,7 @@ pub struct PlanScanObjects + PlanTraceO concurrent: bool, roots: bool, phantom: PhantomData, + bucket: WorkBucketStage, } impl + PlanTraceObject> PlanScanObjects { @@ -995,6 +1029,7 @@ impl + PlanTraceObject> PlanScan buffer: Vec, concurrent: bool, roots: bool, + bucket: WorkBucketStage, ) -> Self { Self { plan, @@ -1002,6 +1037,7 @@ impl + PlanTraceObject> PlanScan concurrent, roots, phantom: PhantomData, + bucket, } } } @@ -1015,12 +1051,16 @@ impl + PlanTraceObject> ScanObje self.roots } + fn get_bucket(&self) -> WorkBucketStage { + self.bucket + } + fn post_scan_object(&self, object: ObjectReference) { self.plan.post_scan_object(object); } fn make_another(&self, buffer: Vec) -> Self { - Self::new(self.plan, buffer, self.concurrent, false) + Self::new(self.plan, buffer, self.concurrent, false, self.bucket) } } @@ -1033,3 +1073,125 @@ impl + PlanTraceObject> GCWork, E: ProcessEdgesWork> { + phantom: PhantomData<(VM, I, E)>, + roots: Vec, + bucket: WorkBucketStage, +} + +impl, E: ProcessEdgesWork> + ProcessRootNode +{ + pub fn new(nodes: Vec, bucket: WorkBucketStage) -> Self { + Self { + phantom: PhantomData, + roots: nodes, + bucket, + } + } +} + +impl, E: ProcessEdgesWork> GCWork + for ProcessRootNode +{ + fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { + trace!("ProcessRootNode"); + + #[cfg(feature = "sanity")] + { + if !mmtk.get_plan().is_in_sanity() { + mmtk.sanity_checker + .lock() + .unwrap() + .add_root_nodes(self.roots.clone()); + } + } + + // Because this is a root packet, the objects in this packet will have not been traced, yet. + // + // This step conceptually traces the edges from root slots to the objects they point to. + // However, VMs that deliver root objects instead of root edges are incapable of updating + // root slots. Like processing an edge, we call `trace_object` on those objects, and + // assert the GC doesn't move those objects because we cannot store back to the slots. + // + // The `scanned_root_objects` variable will hold those root + // objects which are traced for the first time and we create work for scanning those roots. + let scanned_root_objects = { + // We create an instance of E to use its `trace_object` method and its object queue. + let mut process_edges_work = + I::new(vec![], true, mmtk, WorkBucketStage::PinningRootsTrace); + process_edges_work.set_worker(worker); + + for object in self.roots.iter().copied() { + let new_object = process_edges_work.trace_object(object); + debug_assert_eq!( + object, new_object, + "Object moved while tracing root unmovable root object: {} -> {}", + object, new_object + ); + } + + // This contains root objects that are visited the first time. + // It is sufficient to only scan these objects. + process_edges_work.nodes.take() + }; + + let process_edges_work = E::new(vec![], false, mmtk, self.bucket); + let work = process_edges_work.create_scan_work(scanned_root_objects, false); + crate::memory_manager::add_work_packet(mmtk, self.bucket, work); + + trace!("ProcessRootNode End"); + } +} + +/// A `ProcessEdgesWork` type that panics when any of its method is used. +/// This is currently used for plans that do not support transitively pinning. +#[derive(Default)] +pub struct UnsupportedProcessEdges { + phantom: PhantomData, +} + +impl Deref for UnsupportedProcessEdges { + type Target = ProcessEdgesBase; + fn deref(&self) -> &Self::Target { + panic!("unsupported!") + } +} + +impl DerefMut for UnsupportedProcessEdges { + fn deref_mut(&mut self) -> &mut Self::Target { + panic!("unsupported!") + } +} + +impl ProcessEdgesWork for UnsupportedProcessEdges { + type VM = VM; + + type ScanObjectsWorkType = ScanObjects; + + fn new( + _edges: Vec>, + _roots: bool, + _mmtk: &'static MMTK, + _bucket: WorkBucketStage, + ) -> Self { + panic!("unsupported!") + } + + fn trace_object(&mut self, _object: ObjectReference) -> ObjectReference { + panic!("unsupported!") + } + + fn create_scan_work( + &self, + _nodes: Vec, + _roots: bool, + ) -> Self::ScanObjectsWorkType { + panic!("unsupported!") + } +} diff --git a/src/scheduler/scheduler.rs b/src/scheduler/scheduler.rs index 02b6699fe0..083628542d 100644 --- a/src/scheduler/scheduler.rs +++ b/src/scheduler/scheduler.rs @@ -108,15 +108,11 @@ impl GCWorkScheduler { } /// Schedule all the common work packets - pub fn schedule_common_work + 'static>( - &self, - plan: &'static C::PlanType, - ) { + pub fn schedule_common_work>(&self, plan: &'static C::PlanType) { use crate::plan::Plan; use crate::scheduler::gc_work::*; // Stop & scan mutators (mutator scanning can happen before STW) - self.work_buckets[WorkBucketStage::Unconstrained] - .add(StopMutators::::new()); + self.work_buckets[WorkBucketStage::Unconstrained].add(StopMutators::::new()); // Prepare global/collectors/mutators self.work_buckets[WorkBucketStage::Prepare].add(Prepare::::new(plan)); diff --git a/src/scheduler/work.rs b/src/scheduler/work.rs index 86d8aa28a9..6018eed0a3 100644 --- a/src/scheduler/work.rs +++ b/src/scheduler/work.rs @@ -50,10 +50,16 @@ use crate::plan::Plan; /// needs this trait to schedule different work packets. For certain plans, /// they may need to provide several types that implement this trait, e.g. one for /// nursery GC, one for mature GC. -pub trait GCWorkContext { +/// +/// Note: Because `GCWorkContext` is often used as parameters of implementations of `GCWork`, we +/// let GCWorkContext require `Send + 'static`. Since `GCWorkContext` is just a group of +/// associated types, its implementations should not have any actual fields other than +/// `PhantomData`, and will automatically have `Send + 'static`. +pub trait GCWorkContext: Send + 'static { type VM: VMBinding; type PlanType: Plan; // We should use SFTProcessEdges as the default value for this associate type. However, this requires // `associated_type_defaults` which has not yet been stablized. type ProcessEdgesWorkType: ProcessEdgesWork; + type TPProcessEdges: ProcessEdgesWork; } diff --git a/src/scheduler/work_bucket.rs b/src/scheduler/work_bucket.rs index 8569469b2e..b461471081 100644 --- a/src/scheduler/work_bucket.rs +++ b/src/scheduler/work_bucket.rs @@ -232,6 +232,11 @@ pub enum WorkBucketStage { /// Clear the VO bit metadata. Mainly used by ImmixSpace. #[cfg(feature = "vo_bit")] ClearVOBits, + /// Compute the transtive closure starting from transitively pinning (TP) roots following only strong references. + /// No objects in this closure are allow to move. + TPinningClosure, + /// Trace (non-transitively) pinning roots. Objects pointed by those roots must not move, but their children may. To ensure correctness, these must be processed after TPinningClosure + PinningRootsTrace, /// Compute the transtive closure following only strong references. Closure, /// Handle Java-style soft references, and potentially expand the transitive closure. diff --git a/src/util/finalizable_processor.rs b/src/util/finalizable_processor.rs index b15a961014..d341a4694b 100644 --- a/src/util/finalizable_processor.rs +++ b/src/util/finalizable_processor.rs @@ -1,6 +1,6 @@ use crate::plan::is_nursery_gc; use crate::scheduler::gc_work::ProcessEdgesWork; -use crate::scheduler::{GCWork, GCWorker}; +use crate::scheduler::{GCWork, GCWorker, WorkBucketStage}; use crate::util::ObjectReference; use crate::util::VMWorkerThread; use crate::vm::Finalizable; @@ -147,7 +147,7 @@ impl GCWork for Finalization { finalizable_processor.ready_for_finalize.len() ); - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::FinalRefClosure); w.set_worker(worker); finalizable_processor.scan(worker.tls, &mut w, is_nursery_gc(mmtk.get_plan())); debug!( @@ -170,7 +170,7 @@ impl GCWork for ForwardFinalization { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { trace!("Forward finalization"); let mut finalizable_processor = mmtk.finalizable_processor.lock().unwrap(); - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::FinalizableForwarding); w.set_worker(worker); finalizable_processor.forward_candidate(&mut w, is_nursery_gc(mmtk.get_plan())); diff --git a/src/util/reference_processor.rs b/src/util/reference_processor.rs index 4b2709022a..15f540ae49 100644 --- a/src/util/reference_processor.rs +++ b/src/util/reference_processor.rs @@ -6,6 +6,7 @@ use std::vec::Vec; use crate::plan::is_nursery_gc; use crate::scheduler::ProcessEdgesWork; +use crate::scheduler::WorkBucketStage; use crate::util::ObjectReference; use crate::util::VMWorkerThread; use crate::vm::ReferenceGlue; @@ -482,7 +483,7 @@ use std::marker::PhantomData; pub struct SoftRefProcessing(PhantomData); impl GCWork for SoftRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::SoftRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_soft_refs(&mut w, mmtk); w.flush(); @@ -498,7 +499,7 @@ impl SoftRefProcessing { pub struct WeakRefProcessing(PhantomData); impl GCWork for WeakRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::WeakRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_weak_refs(&mut w, mmtk); w.flush(); @@ -514,7 +515,7 @@ impl WeakRefProcessing { pub struct PhantomRefProcessing(PhantomData); impl GCWork for PhantomRefProcessing { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::PhantomRefClosure); w.set_worker(worker); mmtk.reference_processors.scan_phantom_refs(&mut w, mmtk); w.flush(); @@ -530,7 +531,7 @@ impl PhantomRefProcessing { pub struct RefForwarding(PhantomData); impl GCWork for RefForwarding { fn do_work(&mut self, worker: &mut GCWorker, mmtk: &'static MMTK) { - let mut w = E::new(vec![], false, mmtk); + let mut w = E::new(vec![], false, mmtk, WorkBucketStage::RefForwarding); w.set_worker(worker); mmtk.reference_processors.forward_refs(&mut w, mmtk); w.flush(); diff --git a/src/util/sanity/sanity_checker.rs b/src/util/sanity/sanity_checker.rs index f8f2ffdf88..0e4069f7bf 100644 --- a/src/util/sanity/sanity_checker.rs +++ b/src/util/sanity/sanity_checker.rs @@ -89,14 +89,22 @@ impl GCWork for ScheduleSanityGC

{ let sanity_checker = mmtk.sanity_checker.lock().unwrap(); for roots in &sanity_checker.root_edges { scheduler.work_buckets[WorkBucketStage::Closure].add( - SanityGCProcessEdges::::new(roots.clone(), true, mmtk), + SanityGCProcessEdges::::new( + roots.clone(), + true, + mmtk, + WorkBucketStage::Closure, + ), ); } for roots in &sanity_checker.root_nodes { scheduler.work_buckets[WorkBucketStage::Closure].add(ScanObjects::< SanityGCProcessEdges, >::new( - roots.clone(), false, true + roots.clone(), + false, + true, + WorkBucketStage::Closure, )); } } @@ -187,9 +195,14 @@ impl ProcessEdgesWork for SanityGCProcessEdges { type ScanObjectsWorkType = ScanObjects; const OVERWRITE_REFERENCE: bool = false; - fn new(edges: Vec>, roots: bool, mmtk: &'static MMTK) -> Self { + fn new( + edges: Vec>, + roots: bool, + mmtk: &'static MMTK, + bucket: WorkBucketStage, + ) -> Self { Self { - base: ProcessEdgesBase::new(edges, roots, mmtk), + base: ProcessEdgesBase::new(edges, roots, mmtk, bucket), // ..Default::default() } } @@ -238,6 +251,6 @@ impl ProcessEdgesWork for SanityGCProcessEdges { nodes: Vec, roots: bool, ) -> Self::ScanObjectsWorkType { - ScanObjects::::new(nodes, false, roots) + ScanObjects::::new(nodes, false, roots, WorkBucketStage::Closure) } } diff --git a/src/vm/scanning.rs b/src/vm/scanning.rs index a35649fb56..9685c6099f 100644 --- a/src/vm/scanning.rs +++ b/src/vm/scanning.rs @@ -105,18 +105,27 @@ pub trait RootsWorkFactory: Clone + Send + 'static { /// * `edges`: A vector of edges. fn create_process_edge_roots_work(&mut self, edges: Vec); - /// Create work packets to handle nodes pointed by root edges. + /// Create work packets to handle non-transitively pinning roots. /// - /// The work packet cannot update root edges, therefore it cannot move the objects. This - /// method can only be used by GC algorithms that never moves objects, or GC algorithms that - /// supports object pinning. + /// The work packet will prevent the objects in `nodes` from moving, + /// i.e. they will be pinned for the duration of the GC. + /// But it will not prevent the children of those objects from moving. /// /// This method is useful for conservative stack scanning, or VMs that cannot update some /// of the root edges. /// /// Arguments: /// * `nodes`: A vector of references to objects pointed by root edges. - fn create_process_node_roots_work(&mut self, nodes: Vec); + fn create_process_pinning_roots_work(&mut self, nodes: Vec); + + /// Create work packets to handle transitively pinning (TP) roots. + /// + /// Similar to `create_process_pinning_roots_work`, this work packet will not move objects in `nodes`. + /// Unlike ``create_process_pinning_roots_work`, no objects in the transitive closure of `nodes` will be moved, either. + /// + /// Arguments: + /// * `nodes`: A vector of references to objects pointed by root edges. + fn create_process_tpinning_roots_work(&mut self, nodes: Vec); } /// VM-specific methods for scanning roots/objects. diff --git a/tests/test_roots_work_factory.rs b/tests/test_roots_work_factory.rs index 15a95ca227..4a8b64b511 100644 --- a/tests/test_roots_work_factory.rs +++ b/tests/test_roots_work_factory.rs @@ -66,7 +66,11 @@ impl RootsWorkFactory

for MockFactory { } } - fn create_process_node_roots_work(&mut self, _nodes: Vec) { + fn create_process_pinning_roots_work(&mut self, _nodes: Vec) { + unimplemented!(); + } + + fn create_process_tpinning_roots_work(&mut self, _nodes: Vec) { unimplemented!(); } }