Ensure reference objects are either traced or cleared (#1126)

After retaining soft references, we postpone the scanning of soft
references until the transitive closure from retained soft references is
fully expanded. While tracing and scanning new objects reached from the
referents of soft references, new soft references may be discovered, and
they will be added to the reference processor. By postponing the
scanning, we ensure the newly discovered soft references are scanned,
too.

To ensure we do not accidentally enqueue more objects than we should, we
no longer call `trace_object` when *scanning* soft, weak and phantom
references. Instead, we call `ObjectReference::get_forwarded_object` to
get the forwarded references and referents. We still call `trace_object`
when *retaining* soft references, and when forwarding references in
MarkCompact during the second transitive closure.

Fixes: #1125

src/plan/markcompact/global.rs

@@ -112,9 +112,9 @@ impl<VM: VMBinding> Plan for MarkCompact<VM> {
             scheduler.work_buckets[WorkBucketStage::SoftRefClosure]
                 .add(SoftRefProcessing::<MarkingProcessEdges<VM>>::new());
             scheduler.work_buckets[WorkBucketStage::WeakRefClosure]
-                .add(WeakRefProcessing::<MarkingProcessEdges<VM>>::new());
+                .add(WeakRefProcessing::<VM>::new());
             scheduler.work_buckets[WorkBucketStage::PhantomRefClosure]
-                .add(PhantomRefProcessing::<MarkingProcessEdges<VM>>::new());
+                .add(PhantomRefProcessing::<VM>::new());
 
             use crate::util::reference_processor::RefForwarding;
             scheduler.work_buckets[WorkBucketStage::RefForwarding]

src/policy/markcompactspace.rs

@@ -251,7 +251,12 @@ impl<VM: VMBinding> MarkCompactSpace<VM> {
             queue.enqueue(object);
         }
 
-        Self::get_header_forwarding_pointer(object)
+        let result = Self::get_header_forwarding_pointer(object);
+        debug_assert!(
+            !result.is_null(),
+            "Object {object} does not have a forwarding pointer"
+        );
+        result
     }
 
     pub fn test_and_mark(object: ObjectReference) -> bool {

src/scheduler/scheduler.rs

@@ -172,10 +172,9 @@ impl<VM: VMBinding> GCWorkScheduler<VM> {
             };
             self.work_buckets[WorkBucketStage::SoftRefClosure]
                 .add(SoftRefProcessing::<C::DefaultProcessEdges>::new());
-            self.work_buckets[WorkBucketStage::WeakRefClosure]
-                .add(WeakRefProcessing::<C::DefaultProcessEdges>::new());
+            self.work_buckets[WorkBucketStage::WeakRefClosure].add(WeakRefProcessing::<VM>::new());
             self.work_buckets[WorkBucketStage::PhantomRefClosure]
-                .add(PhantomRefProcessing::<C::DefaultProcessEdges>::new());
+                .add(PhantomRefProcessing::<VM>::new());
 
             use crate::util::reference_processor::RefForwarding;
             if plan.constraints().needs_forward_after_liveness {

src/util/finalizable_processor.rs

@@ -1,6 +1,7 @@
 use crate::plan::is_nursery_gc;
 use crate::scheduler::gc_work::ProcessEdgesWork;
 use crate::scheduler::{GCWork, GCWorker, WorkBucketStage};
+use crate::util::reference_processor::RescanReferences;
 use crate::util::ObjectReference;
 use crate::util::VMWorkerThread;
 use crate::vm::Finalizable;
@@ -140,6 +141,17 @@ pub struct Finalization<E: ProcessEdgesWork>(PhantomData<E>);
 
 impl<E: ProcessEdgesWork> GCWork<E::VM> for Finalization<E> {
     fn do_work(&mut self, worker: &mut GCWorker<E::VM>, mmtk: &'static MMTK<E::VM>) {
+        if !*mmtk.options.no_reference_types {
+            // Rescan soft and weak references at the end of the transitive closure from resurrected
+            // objects.  New soft and weak references may be discovered during this.
+            let rescan = Box::new(RescanReferences {
+                soft: true,
+                weak: true,
+                phantom_data: PhantomData,
+            });
+            worker.scheduler().work_buckets[WorkBucketStage::FinalRefClosure].set_sentinel(rescan);
+        }
+
         let mut finalizable_processor = mmtk.finalizable_processor.lock().unwrap();
         debug!(
             "Finalization, {} objects in candidates, {} objects ready to finalize",