8353115: GenShen: mixed evacuation candidate regions need accurate live_data

src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp

@@ -129,6 +129,13 @@ class ShenandoahHeuristics : public CHeapObj<mtGC> {
 #endif
     }
 
+    inline void update_livedata(size_t live) {
+      _region_union._live_data = live;
+#ifdef ASSERT
+      _union_tag = is_live_data;
+#endif
+    }
+
     inline ShenandoahHeapRegion* get_region() const {
       assert(_union_tag != is_uninitialized, "Cannot fetch region from uninitialized RegionData");
       return _region;

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp

@@ -88,6 +88,17 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll
     return false;
   }
 
+  // Between consecutive mixed-evacuation cycles, the live data within each candidate region may change due to
+  // promotions and old-gen evacuations.  Re-sort the candidate regions in order to first evacuate regions that have
+  // the smallest amount of live data.  These are easiest to evacuate with least effort.  Doing these first allows
+  // us to more quickly replenish free memory with empty regions.
+  for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; i++) {
+    ShenandoahHeapRegion* r = _region_data[i].get_region();
+    _region_data[i].update_livedata(r->get_live_data_bytes());
+  }
+  QuickSort::sort<RegionData>(_region_data + _next_old_collection_candidate, unprocessed_old_collection_candidates(),
+                              compare_by_live);
+
   _first_pinned_candidate = NOT_FOUND;
 
   uint included_old_regions = 0;
@@ -312,6 +323,19 @@ void ShenandoahOldHeuristics::slide_pinned_regions_to_front() {
   _next_old_collection_candidate = write_index + 1;
 }
 
+void ShenandoahOldHeuristics::recalibrate_old_collection_candidates_live_memory() {
+  size_t total_live_data = 0;
+  for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; i++) {
+    ShenandoahHeapRegion* r = _region_data[i].get_region();
+    size_t region_live = r->get_live_data_bytes();
+    total_live_data += region_live;
+    _region_data[i].update_livedata(region_live);
+  }
+  QuickSort::sort<RegionData>(_region_data + _next_old_collection_candidate, unprocessed_old_collection_candidates(),
+                              compare_by_live);
+  _live_bytes_in_unprocessed_candidates = total_live_data;
+}
+
 void ShenandoahOldHeuristics::prepare_for_old_collections() {
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 

src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp

@@ -131,6 +131,13 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics {
   // Return true iff the collection set is primed with at least one old-gen region.
   bool prime_collection_set(ShenandoahCollectionSet* set);
 
+  // Between consecutive mixed-evacuation cycles, the live data within each candidate region may change due to
+  // promotions and old-gen evacuations.  Recompute live data in candidates and re--sort the candidate regions in
+  // order to first evacuate regions that have the smallest amount of live data.  These are easiest to evacuate
+  // with least effort.  Doing these first allows us to more quickly replenish free memory with entirely empty regions.
+  // Call this during final mark before adjusting the old-evacuation ratio.
+  void recalibrate_old_collection_candidates_live_memory();
+
   // How many old-collection candidates have not yet been processed?
   uint unprocessed_old_collection_candidates() const;
 

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -653,7 +653,7 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl
   ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {}
 
   void heap_region_do(ShenandoahHeapRegion* r) {
-    assert(!r->has_live(), "Region %zu should have no live data", r->index());
+    assert(!r->has_marked(), "Region %zu should have no marked data", r->index());
     if (r->is_active()) {
       // Check if region needs updating its TAMS. We have updated it already during concurrent
       // reset, so it is very likely we don't need to do another write here.  Since most regions

src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp

@@ -925,12 +925,14 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
 private:
   ShenandoahHeap* const _heap;
   bool _is_generational;
+  ShenandoahMarkingContext* _mark_context;
   size_t _young_regions, _young_usage, _young_humongous_waste;
   size_t _old_regions, _old_usage, _old_humongous_waste;
 
 public:
   ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()),
                                    _is_generational(_heap->mode()->is_generational()),
+                                   _mark_context(_heap->complete_marking_context()),
                                    _young_regions(0),
                                    _young_usage(0),
                                    _young_humongous_waste(0),
@@ -983,7 +985,7 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure {
         ShenandoahGenerationalFullGC::account_for_region(r, _young_regions, _young_usage, _young_humongous_waste);
       }
     }
-    r->set_live_data(live);
+    r->set_live_data_after_fullgc(live);
     r->reset_alloc_metadata();
   }
 
@@ -1093,7 +1095,7 @@ class ShenandoahMCResetCompleteBitmapTask : public WorkerTask {
     ShenandoahHeap* heap = ShenandoahHeap::heap();
     ShenandoahMarkingContext* const ctx = heap->complete_marking_context();
     while (region != nullptr) {
-      if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_live()) {
+      if (heap->is_bitmap_slice_committed(region) && !region->is_pinned() && region->has_marked()) {
         ctx->clear_bitmap(region);
       }
       region = _regions.next();

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -2561,6 +2561,10 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) {
     // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this
     // available for transfer to old. Note that transfer of humongous regions does not impact available.
     ShenandoahGenerationalHeap* gen_heap = ShenandoahGenerationalHeap::heap();
+    ShenandoahOldGeneration* old_gen = gen_heap->old_generation();
+    ShenandoahOldHeuristics* old_heuristics = old_gen->heuristics();
+
+    old_heuristics->recalibrate_old_collection_candidates_live_memory();
     size_t allocation_runway = gen_heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions);
     gen_heap->compute_old_generation_balance(allocation_runway, old_cset_regions);
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -75,6 +75,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
   _plab_allocs(0),
   _live_data(0),
   _critical_pins(0),
+  _mixed_candidate_garbage_words(0),
   _update_watermark(start),
   _age(0),
 #ifdef SHENANDOAH_CENSUS_NOISE
@@ -391,9 +392,11 @@ size_t ShenandoahHeapRegion::get_plab_allocs() const {
   return _plab_allocs * HeapWordSize;
 }
 
-void ShenandoahHeapRegion::set_live_data(size_t s) {
+void ShenandoahHeapRegion::set_live_data_after_fullgc(size_t s) {
   assert(Thread::current()->is_VM_thread(), "by VM thread");
-  _live_data = (s >> LogHeapWordSize);
+  ShenandoahMarkingContext* marking_context = ShenandoahHeap::heap()->complete_marking_context();
+  size_t words_allocated_above_tams = pointer_delta(top(), marking_context->top_at_mark_start(this));
+  _live_data = (s >> LogHeapWordSize) - words_allocated_above_tams;
 }
 
 void ShenandoahHeapRegion::print_on(outputStream* st) const {

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -259,6 +259,8 @@ class ShenandoahHeapRegion {
   volatile size_t _live_data;
   volatile size_t _critical_pins;
 
+  size_t _mixed_candidate_garbage_words;
+
   HeapWord* volatile _update_watermark;
 
   uint _age;
@@ -367,18 +369,28 @@ class ShenandoahHeapRegion {
   inline HeapWord* allocate(size_t word_size, const ShenandoahAllocRequest& req);
 
   inline void clear_live_data();
-  void set_live_data(size_t s);
-
-  // Increase live data for newly allocated region
-  inline void increase_live_data_alloc_words(size_t s);
+  void set_live_data_after_fullgc(size_t s);
 
   // Increase live data for region scanned with GC
   inline void increase_live_data_gc_words(size_t s);
 
+  inline bool has_marked() const;
+
   inline bool has_live() const;
+
+  // Returns bytes identified as live by most recently completed marking effort.  Can only be called during safepoints.
+  inline size_t get_marked_data_bytes() const;
+
+  // Returns bytes identified as live by most recently completed marking effort, plus allocations above TAMS.
+  // Can only be called during safepoints.
   inline size_t get_live_data_bytes() const;
+
+  // Returns words identified as live by most recently completed marking effort, plus allocations above TAMS.
+  // Can only be called during safepoints.
   inline size_t get_live_data_words() const;
 
+  // Returns garbage by calculating difference between used and get_live_data_words.  Can only be called at
+  // safepoints. Allocations above TAMS are considered live.
   inline size_t garbage() const;
 
   void print_on(outputStream* st) const;

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp

@@ -128,10 +128,6 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest::
   }
 }
 
-inline void ShenandoahHeapRegion::increase_live_data_alloc_words(size_t s) {
-  internal_increase_live_data(s);
-}
-
 inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) {
   internal_increase_live_data(s);
   if (ShenandoahPacing) {
@@ -147,14 +143,25 @@ inline void ShenandoahHeapRegion::clear_live_data() {
   Atomic::store(&_live_data, (size_t)0);
 }
 
+inline size_t ShenandoahHeapRegion::get_marked_data_bytes() const {
+  return Atomic::load(&_live_data) * HeapWordSize;
+}
+
 inline size_t ShenandoahHeapRegion::get_live_data_words() const {
-  return Atomic::load(&_live_data);
+  ShenandoahMarkingContext *ctx = ShenandoahHeap::heap()->complete_marking_context();
+  HeapWord* tams = ctx->top_at_mark_start(this);
+  size_t words_above_tams = pointer_delta(top(), tams);
+  return Atomic::load(&_live_data) + words_above_tams;
 }
 
 inline size_t ShenandoahHeapRegion::get_live_data_bytes() const {
   return get_live_data_words() * HeapWordSize;
 }
 
+inline bool ShenandoahHeapRegion::has_marked() const {
+  return Atomic::load(&_live_data) != 0;
+}
+
 inline bool ShenandoahHeapRegion::has_live() const {
   return get_live_data_words() != 0;
 }
@@ -163,20 +170,21 @@ inline size_t ShenandoahHeapRegion::garbage() const {
   assert(used() >= get_live_data_bytes(),
          "Live Data must be a subset of used() live: %zu used: %zu",
          get_live_data_bytes(), used());
-
   size_t result = used() - get_live_data_bytes();
   return result;
 }
 
 inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const {
   assert(get_top_before_promote() != nullptr, "top before promote should not equal null");
   size_t used_before_promote = byte_size(bottom(), get_top_before_promote());
-  assert(used_before_promote >= get_live_data_bytes(),
+  assert(used_before_promote >= get_marked_data_bytes(),
          "Live Data must be a subset of used before promotion live: %zu used: %zu",
-         get_live_data_bytes(), used_before_promote);
-  size_t result = used_before_promote - get_live_data_bytes();
+         get_marked_data_bytes(), used_before_promote);
+  ShenandoahMarkingContext *ctx = ShenandoahHeap::heap()->complete_marking_context();
+  HeapWord* tams = ctx->top_at_mark_start(this);
+  size_t bytes_allocated_during_mark = pointer_delta(get_top_before_promote(), tams) * HeapWordSize;
+  size_t result = used_before_promote - (get_marked_data_bytes() + bytes_allocated_during_mark);
   return result;
-
 }
 
 inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const {

src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp

@@ -57,21 +57,6 @@ ShenandoahFinalMarkUpdateRegionStateClosure::ShenandoahFinalMarkUpdateRegionStat
 
 void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapRegion* r) {
   if (r->is_active()) {
-    if (_ctx != nullptr) {
-      // _ctx may be null when this closure is used to sync only the pin status
-      // update the watermark of old regions. For old regions we cannot reset
-      // the TAMS because we rely on that to keep promoted objects alive after
-      // old marking is complete.
-
-      // All allocations past TAMS are implicitly live, adjust the region data.
-      // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap.
-      HeapWord *tams = _ctx->top_at_mark_start(r);
-      HeapWord *top = r->top();
-      if (top > tams) {
-        r->increase_live_data_alloc_words(pointer_delta(top, tams));
-      }
-    }
-
     // We are about to select the collection set, make sure it knows about
     // current pinning status. Also, this allows trashing more regions that
     // now have their pinning status dropped.

test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp

@@ -104,7 +104,7 @@ class ShenandoahOldHeuristicTest : public ::testing::Test {
     region->set_affiliation(OLD_GENERATION);
     region->make_regular_allocation(OLD_GENERATION);
     size_t live_bytes = ShenandoahHeapRegion::region_size_bytes() - garbage_bytes;
-    region->increase_live_data_alloc_words(live_bytes / HeapWordSize);
+    region->increase_live_data_gc_words(live_bytes / HeapWordSize);
     region->set_top(region->end());
     return region->garbage();
   }