8369048: GenShen: Defer ShenFreeSet::available() during rebuild

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp

@@ -28,8 +28,12 @@
 
 #include "gc/shenandoah/shenandoahHeap.hpp"
 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
+#include "gc/shenandoah/shenandoahLock.hpp"
 #include "gc/shenandoah/shenandoahSimpleBitMap.hpp"
 
+typedef ShenandoahLock    ShenandoahRebuildLock;
+typedef ShenandoahLocker  ShenandoahRebuildLocker;
+
 // Each ShenandoahHeapRegion is associated with a ShenandoahFreeSetPartitionId.
 enum class ShenandoahFreeSetPartitionId : uint8_t {
   Mutator,                      // Region is in the Mutator free set: available memory is available to mutators.
@@ -234,7 +238,7 @@ class ShenandoahRegionPartitions {
   // Return available_in assuming caller does not hold the heap lock.  In production builds, available is
   // returned without acquiring the lock.  In debug builds, the global heap lock is acquired in order to
   // enforce a consistency assert.
-  inline size_t available_in_not_locked(ShenandoahFreeSetPartitionId which_partition) const {
+  inline size_t available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId which_partition) const {
     assert (which_partition < NumPartitions, "selected free set must be valid");
     shenandoah_assert_not_heaplocked();
 #ifdef ASSERT
@@ -316,6 +320,9 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
   ShenandoahHeap* const _heap;
   ShenandoahRegionPartitions _partitions;
 
+  // This locks the rebuild process (in combination with the global heap lock)
+  ShenandoahRebuildLock _lock;
+
   HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r);
 
   // Return the address of memory allocated, setting in_new_region to true iff the allocation is taken
@@ -415,6 +422,11 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
 
   ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions);
 
+
+  ShenandoahRebuildLock* lock() {
+    return &_lock;
+  }
+
   // Public because ShenandoahRegionPartitions assertions require access.
   inline size_t alloc_capacity(ShenandoahHeapRegion *r) const;
   inline size_t alloc_capacity(size_t idx) const;
@@ -480,7 +492,10 @@ class ShenandoahFreeSet : public CHeapObj<mtGC> {
   // locked action can be seen by these unlocked functions.
   inline size_t capacity()  const { return _partitions.capacity_of(ShenandoahFreeSetPartitionId::Mutator);             }
   inline size_t used()      const { return _partitions.used_by(ShenandoahFreeSetPartitionId::Mutator);                 }
-  inline size_t available() const { return _partitions.available_in_not_locked(ShenandoahFreeSetPartitionId::Mutator); }
+  inline size_t available() {
+    ShenandoahRebuildLocker locker(lock());
+    return _partitions.available_in_locked_for_rebuild(ShenandoahFreeSetPartitionId::Mutator);
+  }
 
   HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region);
 

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -1153,17 +1153,17 @@ ShenandoahGenerationalHeap::TransferResult ShenandoahFullGC::phase5_epilog() {
     }
 
     heap->collection_set()->clear();
-    size_t young_cset_regions, old_cset_regions;
-    size_t first_old, last_old, num_old;
-    heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
-
+    size_t young_cset_regions, old_cset_regions, first_old, last_old, num_old;
+    ShenandoahFreeSet* free_set = heap->free_set();
+    ShenandoahRebuildLocker rebuild_locker(free_set->lock());
+    free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
     // We also do not expand old generation size following Full GC because we have scrambled age populations and
     // no longer have objects separated by age into distinct regions.
     if (heap->mode()->is_generational()) {
       ShenandoahGenerationalFullGC::compute_balances();
     }
 
-    heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
+    free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
 
     // Set mark incomplete because the marking bitmaps have been reset except pinned regions.
     heap->global_generation()->set_mark_incomplete();

src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp

@@ -745,14 +745,16 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) {
   {
     ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset :
                             ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
+    ShenandoahFreeSet* free_set = heap->free_set();
     ShenandoahHeapLocker locker(heap->lock());
+    ShenandoahRebuildLocker rebuild_locker(free_set->lock());
     size_t young_cset_regions, old_cset_regions;
 
     // We are preparing for evacuation.  At this time, we ignore cset region tallies.
     size_t first_old, last_old, num_old;
-    heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
+    free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
     // Free set construction uses reserve quantities, because they are known to be valid here
-    heap->free_set()->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true);
+    free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old, true);
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -424,6 +424,7 @@ jint ShenandoahHeap::initialize() {
 
     // We are initializing free set.  We ignore cset region tallies.
     size_t first_old, last_old, num_old;
+    ShenandoahRebuildLocker rebuild_locker(_free_set->lock());
     _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old, last_old, num_old);
     _free_set->finish_rebuild(young_cset_regions, old_cset_regions, num_old);
   }
@@ -2575,6 +2576,7 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) {
                           ShenandoahPhaseTimings::final_update_refs_rebuild_freeset :
                           ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset);
   ShenandoahHeapLocker locker(lock());
+  ShenandoahRebuildLocker rebuild_locker(_free_set->lock());
   size_t young_cset_regions, old_cset_regions;
   size_t first_old_region, last_old_region, old_region_count;
   _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions, first_old_region, last_old_region, old_region_count);
@@ -2600,8 +2602,8 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) {
 
     // Total old_available may have been expanded to hold anticipated promotions.  We trigger if the fragmented available
     // memory represents more than 16 regions worth of data.  Note that fragmentation may increase when we promote regular
-    // regions in place when many of these regular regions have an abundant amount of available memory within them.  Fragmentation
-    // will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
+    // regions in place when many of these regular regions have an abundant amount of available memory within them.
+    // Fragmentation will decrease as promote-by-copy consumes the available memory within these partially consumed regions.
     //
     // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides
     // within partially consumed regions of memory.

src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp

@@ -509,13 +509,14 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent
     ShenandoahGCPhase phase(concurrent ?
         ShenandoahPhaseTimings::final_rebuild_freeset :
         ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset);
+    ShenandoahFreeSet* free_set = heap->free_set();
     ShenandoahHeapLocker locker(heap->lock());
-    size_t cset_young_regions, cset_old_regions;
-    size_t first_old, last_old, num_old;
-    heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old);
-    // This is just old-gen completion.  No future budgeting required here.  The only reason to rebuild the freeset here
-    // is in case there was any immediate old garbage identified.
-    heap->free_set()->finish_rebuild(cset_young_regions, cset_old_regions, num_old);
+    ShenandoahRebuildLocker rebuild_locker(free_set->lock());
+    size_t cset_young_regions, cset_old_regions, first_old, last_old, num_old;
+    free_set->prepare_to_rebuild(cset_young_regions, cset_old_regions, first_old, last_old, num_old);
+    // This is completion of old-gen marking.  We rebuild in order to reclaim immediate garbage and to
+    // prepare for subsequent mixed evacuations.
+    free_set->finish_rebuild(cset_young_regions, cset_old_regions, num_old);
   }
 }