diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index b906ae2ca0b..dbe3f0b899d 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -826,11 +826,6 @@ bool ShenandoahBarrierSetC2::optimize_loops(PhaseIdealLoop* phase, LoopOptsMode assert(UseShenandoahGC, "only for shenandoah"); ShenandoahBarrierC2Support::pin_and_expand(phase); return true; - } else if (mode == LoopOptsShenandoahPostExpand) { - assert(UseShenandoahGC, "only for shenandoah"); - visited.clear(); - ShenandoahBarrierC2Support::optimize_after_expansion(visited, nstack, worklist, phase); - return true; } return false; } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index c1acde2118e..602c6635e69 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -122,8 +122,8 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 { virtual Node* step_over_gc_barrier(Node* c) const; virtual bool expand_barriers(Compile* C, PhaseIterGVN& igvn) const; virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const; - virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; } - virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand || mode == LoopOptsShenandoahPostExpand; } + virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand; } + virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return mode == LoopOptsShenandoahExpand; } // Support for macro expanded GC barriers virtual void register_potential_barrier_node(Node* node) const; diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 831b72066e2..98ac0fb0a2a 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -52,12 +52,6 @@ bool ShenandoahBarrierC2Support::expand(Compile* C, PhaseIterGVN& igvn) { C->clear_major_progress(); PhaseIdealLoop::optimize(igvn, LoopOptsShenandoahExpand); if (C->failing()) return false; - - C->set_major_progress(); - if (!C->optimize_loops(igvn, LoopOptsShenandoahPostExpand)) { - return false; - } - C->clear_major_progress(); C->process_for_post_loop_opts_igvn(igvn); if (C->failing()) return false; @@ -1505,236 +1499,6 @@ Node* ShenandoahBarrierC2Support::get_load_addr(PhaseIdealLoop* phase, VectorSet } -void ShenandoahBarrierC2Support::move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase) { - IdealLoopTree *loop = phase->get_loop(iff); - Node* loop_head = loop->_head; - Node* entry_c = loop_head->in(LoopNode::EntryControl); - - Node* bol = iff->in(1); - Node* cmp = bol->in(1); - Node* andi = cmp->in(1); - Node* load = andi->in(1); - - assert(is_gc_state_load(load), "broken"); - if (!phase->is_dominator(load->in(0), entry_c)) { - Node* mem_ctrl = nullptr; - Node* mem = dom_mem(load->in(MemNode::Memory), loop_head, Compile::AliasIdxRaw, mem_ctrl, phase); - load = load->clone(); - load->set_req(MemNode::Memory, mem); - load->set_req(0, entry_c); - phase->register_new_node(load, entry_c); - andi = andi->clone(); - andi->set_req(1, load); - phase->register_new_node(andi, entry_c); - cmp = cmp->clone(); - cmp->set_req(1, andi); - phase->register_new_node(cmp, entry_c); - bol = bol->clone(); - bol->set_req(1, cmp); - phase->register_new_node(bol, entry_c); - - phase->igvn().replace_input_of(iff, 1, bol); - } -} - -bool ShenandoahBarrierC2Support::identical_backtoback_ifs(Node* n, PhaseIdealLoop* phase) { - if (!n->is_If() || n->is_CountedLoopEnd()) { - return false; - } - Node* region = n->in(0); - - if (!region->is_Region()) { - return false; - } - Node* dom = phase->idom(region); - if (!dom->is_If()) { - return false; - } - - if (!is_heap_stable_test(n) || !is_heap_stable_test(dom)) { - return false; - } - - IfNode* dom_if = dom->as_If(); - Node* proj_true = dom_if->proj_out(1); - Node* proj_false = dom_if->proj_out(0); - - for (uint i = 1; i < region->req(); i++) { - if (phase->is_dominator(proj_true, region->in(i))) { - continue; - } - if (phase->is_dominator(proj_false, region->in(i))) { - continue; - } - return false; - } - - return true; -} - -bool ShenandoahBarrierC2Support::merge_point_safe(Node* region) { - for (DUIterator_Fast imax, i = region->fast_outs(imax); i < imax; i++) { - Node* n = region->fast_out(i); - if (n->is_LoadStore()) { - // Splitting a LoadStore node through phi, causes it to lose its SCMemProj: the split if code doesn't have support - // for a LoadStore at the region the if is split through because that's not expected to happen (LoadStore nodes - // should be between barrier nodes). It does however happen with Shenandoah though because barriers can get - // expanded around a LoadStore node. - return false; - } - } - return true; -} - - -void ShenandoahBarrierC2Support::merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase) { - assert(is_heap_stable_test(n), "no other tests"); - if (identical_backtoback_ifs(n, phase)) { - Node* n_ctrl = n->in(0); - if (phase->can_split_if(n_ctrl) && merge_point_safe(n_ctrl)) { - IfNode* dom_if = phase->idom(n_ctrl)->as_If(); - if (is_heap_stable_test(n)) { - Node* gc_state_load = n->in(1)->in(1)->in(1)->in(1); - assert(is_gc_state_load(gc_state_load), "broken"); - Node* dom_gc_state_load = dom_if->in(1)->in(1)->in(1)->in(1); - assert(is_gc_state_load(dom_gc_state_load), "broken"); - if (gc_state_load != dom_gc_state_load) { - phase->igvn().replace_node(gc_state_load, dom_gc_state_load); - } - } - PhiNode* bolphi = PhiNode::make_blank(n_ctrl, n->in(1)); - Node* proj_true = dom_if->proj_out(1); - Node* proj_false = dom_if->proj_out(0); - Node* con_true = phase->igvn().makecon(TypeInt::ONE); - Node* con_false = phase->igvn().makecon(TypeInt::ZERO); - - for (uint i = 1; i < n_ctrl->req(); i++) { - if (phase->is_dominator(proj_true, n_ctrl->in(i))) { - bolphi->init_req(i, con_true); - } else { - assert(phase->is_dominator(proj_false, n_ctrl->in(i)), "bad if"); - bolphi->init_req(i, con_false); - } - } - phase->register_new_node(bolphi, n_ctrl); - phase->igvn().replace_input_of(n, 1, bolphi); - phase->do_split_if(n); - } - } -} - -IfNode* ShenandoahBarrierC2Support::find_unswitching_candidate(const IdealLoopTree* loop, PhaseIdealLoop* phase) { - // Find first invariant test that doesn't exit the loop - LoopNode *head = loop->_head->as_Loop(); - IfNode* unswitch_iff = nullptr; - Node* n = head->in(LoopNode::LoopBackControl); - int loop_has_sfpts = -1; - while (n != head) { - Node* n_dom = phase->idom(n); - if (n->is_Region()) { - if (n_dom->is_If()) { - IfNode* iff = n_dom->as_If(); - if (iff->in(1)->is_Bool()) { - BoolNode* bol = iff->in(1)->as_Bool(); - if (bol->in(1)->is_Cmp()) { - // If condition is invariant and not a loop exit, - // then found reason to unswitch. - if (is_heap_stable_test(iff) && - (loop_has_sfpts == -1 || loop_has_sfpts == 0)) { - assert(!loop->is_loop_exit(iff), "both branches should be in the loop"); - if (loop_has_sfpts == -1) { - for(uint i = 0; i < loop->_body.size(); i++) { - Node *m = loop->_body[i]; - if (m->is_SafePoint() && !m->is_CallLeaf()) { - loop_has_sfpts = 1; - break; - } - } - if (loop_has_sfpts == -1) { - loop_has_sfpts = 0; - } - } - if (!loop_has_sfpts) { - unswitch_iff = iff; - } - } - } - } - } - } - n = n_dom; - } - return unswitch_iff; -} - - -void ShenandoahBarrierC2Support::optimize_after_expansion(VectorSet &visited, Node_Stack &stack, Node_List &old_new, PhaseIdealLoop* phase) { - Node_List heap_stable_tests; - stack.push(phase->C->start(), 0); - do { - Node* n = stack.node(); - uint i = stack.index(); - - if (i < n->outcnt()) { - Node* u = n->raw_out(i); - stack.set_index(i+1); - if (!visited.test_set(u->_idx)) { - stack.push(u, 0); - } - } else { - stack.pop(); - if (n->is_If() && is_heap_stable_test(n)) { - heap_stable_tests.push(n); - } - } - } while (stack.size() > 0); - - for (uint i = 0; i < heap_stable_tests.size(); i++) { - Node* n = heap_stable_tests.at(i); - assert(is_heap_stable_test(n), "only evacuation test"); - merge_back_to_back_tests(n, phase); - } - - if (!phase->C->major_progress()) { - VectorSet seen; - for (uint i = 0; i < heap_stable_tests.size(); i++) { - Node* n = heap_stable_tests.at(i); - IdealLoopTree* loop = phase->get_loop(n); - if (loop != phase->ltree_root() && - loop->_child == nullptr && - !loop->_irreducible) { - Node* head = loop->_head; - if (head->is_Loop() && - (!head->is_CountedLoop() || head->as_CountedLoop()->is_main_loop() || head->as_CountedLoop()->is_normal_loop()) && - !seen.test_set(head->_idx)) { - IfNode* iff = find_unswitching_candidate(loop, phase); - if (iff != nullptr) { - Node* bol = iff->in(1); - if (head->as_Loop()->is_strip_mined()) { - head->as_Loop()->verify_strip_mined(0); - } - move_gc_state_test_out_of_loop(iff, phase); - - AutoNodeBudget node_budget(phase); - - if (loop->policy_unswitching(phase)) { - if (head->as_Loop()->is_strip_mined()) { - OuterStripMinedLoopNode* outer = head->as_CountedLoop()->outer_loop(); - hide_strip_mined_loop(outer, head->as_CountedLoop(), phase); - } - phase->do_unswitching(loop, old_new); - } else { - // Not proceeding with unswitching. Move load back in - // the loop. - phase->igvn().replace_input_of(iff, 1, bol); - } - } - } - } - } - } -} - #ifdef ASSERT static bool has_never_branch(Node* root) { for (uint i = 1; i < root->req(); i++) { diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp index 164502e6358..93572cddc5b 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp @@ -63,12 +63,7 @@ class ShenandoahBarrierC2Support : public AllStatic { static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, DecoratorSet decorators, PhaseIdealLoop* phase); static void test_in_cset(Node*& ctrl, Node*& not_cset_ctrl, Node* val, Node* raw_mem, PhaseIdealLoop* phase); - static void move_gc_state_test_out_of_loop(IfNode* iff, PhaseIdealLoop* phase); - static void merge_back_to_back_tests(Node* n, PhaseIdealLoop* phase); - static bool merge_point_safe(Node* region); - static bool identical_backtoback_ifs(Node *n, PhaseIdealLoop* phase); static void fix_ctrl(Node* barrier, Node* region, const MemoryGraphFixer& fixer, Unique_Node_List& uses, Unique_Node_List& uses_to_ignore, uint last, PhaseIdealLoop* phase); - static IfNode* find_unswitching_candidate(const IdealLoopTree *loop, PhaseIdealLoop* phase); static Node* get_load_addr(PhaseIdealLoop* phase, VectorSet& visited, Node* lrb); public: @@ -80,7 +75,6 @@ class ShenandoahBarrierC2Support : public AllStatic { static bool expand(Compile* C, PhaseIterGVN& igvn); static void pin_and_expand(PhaseIdealLoop* phase); - static void optimize_after_expansion(VectorSet& visited, Node_Stack& nstack, Node_List& old_new, PhaseIdealLoop* phase); #ifdef ASSERT static void verify(RootNode* root); diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 7c251bd4bd9..6992dc1702d 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -104,7 +104,6 @@ enum LoopOptsMode { LoopOptsNone, LoopOptsMaxUnroll, LoopOptsShenandoahExpand, - LoopOptsShenandoahPostExpand, LoopOptsSkipSplitIf, LoopOptsVerify }; diff --git a/test/hotspot/jtreg/compiler/gcbarriers/TestShenandoahBarrierExpansion.java b/test/hotspot/jtreg/compiler/gcbarriers/TestShenandoahBarrierExpansion.java index 84a34695092..6a1431c43df 100644 --- a/test/hotspot/jtreg/compiler/gcbarriers/TestShenandoahBarrierExpansion.java +++ b/test/hotspot/jtreg/compiler/gcbarriers/TestShenandoahBarrierExpansion.java @@ -71,7 +71,7 @@ private static int testLoadTwoObjectFieldsWithNullCheck() { @Test @IR(failOn = IRNode.IF, phase = CompilePhase.AFTER_PARSING) - @IR(counts = { IRNode.IF, "3" }, phase = CompilePhase.BARRIER_EXPANSION) + @IR(counts = { IRNode.IF, "4" }, phase = CompilePhase.BARRIER_EXPANSION) private static void testLoadTwoFieldObjectAndEscape() { final A field2 = staticField2; final A field3 = staticField3;