WIP: Fix compilation for AArch64.

Author: wks

openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.cpp

@@ -25,14 +25,16 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "interpreter/interp_masm.hpp"
 #include "mmtkBarrierSet.hpp"
-#include "mmtkBarrierSetAssembler_aarch64.hpp"
 #include "mmtkBarrierSetC1.hpp"
 #include "mmtkMutator.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/macros.hpp"
 #include "c1/c1_LIRAssembler.hpp"
 #include "c1/c1_MacroAssembler.hpp"
 
+#include "utilities/macros.hpp"
+#include CPU_HEADER(mmtkBarrierSetAssembler)
+
 #define __ masm->
 
 void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj, Register var_size_in_bytes, int con_size_in_bytes, Register tmp1, Register tmp2, Label& slow_case) {
@@ -129,61 +131,95 @@ void MMTkBarrierSetAssembler::eden_allocate(MacroAssembler* masm, Register obj,
 
 #undef __
 
-#define __ sasm->
-
-void MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) const {
-  // printf("xxx MMTkBarrierSetAssembler::generate_c1_write_barrier_runtime_stub\n");
-  // See also void G1BarrierSetAssembler::generate_c1_post_barrier_runtime_stub(StubAssembler* sasm)
-  __ prologue("mmtk_write_barrier", false);
+//////////////////// Assembler for C1 ////////////////////
 
-  Label done, runtime;
+// Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1
 
-  // void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg)
-  // ld(reg, Address(fp, offset_in_words * BytesPerWord));
-  // ra is free to use here, because call prologue/epilogue handles it
-  // Zheyuan: Code works by swaping rscratch2 and rscratch1, and I dont know why
-  const Register src = rscratch2;
-  const Register slot = rscratch1;
-  const Register new_val = lr;
-  __ load_parameter(0, src);
-  __ load_parameter(1, slot);
-  __ load_parameter(2, new_val);
-
-  __ bind(runtime);
+#define __ sasm->
 
-  // Push integer registers x7, x10-x17, x28-x31.
-  //                        t2, a0-a7,   t3-t6
+void MMTkBarrierSetAssembler::generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address entry_point, int argc) {
+  __ prologue(name, false);
   __ push_call_clobbered_registers();
 
-  if (mmtk_enable_barrier_fastpath) {
-    __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), src, slot, new_val);
-  } else {
-    __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), src, slot, new_val);
+  if (argc > 0) __ load_parameter(0, c_rarg0);
+  if (argc > 1) __ load_parameter(1, c_rarg1);
+  if (argc > 2) __ load_parameter(2, c_rarg2);
+  if (argc > 3) {
+    guarantee(false, "Too many args");
   }
 
+  __ call_VM_leaf(entry_point, 3);
+
   __ pop_call_clobbered_registers();
+  __ epilogue();
+}
 
-  __ bind(done);
+// void foo(){
+//   Label done, runtime;
 
-  __ epilogue();
+//   // void C1_MacroAssembler::load_parameter(int offset_in_words, Register reg)
+//   // ld(reg, Address(fp, offset_in_words * BytesPerWord));
+//   // ra is free to use here, because call prologue/epilogue handles it
+//   // Zheyuan: Code works by swaping rscratch2 and rscratch1, and I dont know why
+//   const Register src = rscratch2;
+//   const Register slot = rscratch1;
+//   const Register new_val = lr;
+//   __ load_parameter(0, src);
+//   __ load_parameter(1, slot);
+//   __ load_parameter(2, new_val);
+
+//   __ bind(runtime);
+
+//   // Push integer registers x7, x10-x17, x28-x31.
+//   //                        t2, a0-a7,   t3-t6
+//   __ push_call_clobbered_registers();
+
+//   if (mmtk_enable_barrier_fastpath) {
+//     __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), src, slot, new_val);
+//   } else {
+//     __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), src, slot, new_val);
+//   }
+
+//   __ pop_call_clobbered_registers();
+
+//   __ bind(done);
+
+//   __ epilogue();
+// }
+
+void MMTkBarrierSetAssembler::generate_c1_load_reference_runtime_stub(StubAssembler* sasm) {
+  generate_c1_runtime_stub_general(sasm, "c1_load_reference_runtime_stub", FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1);
+}
+
+void MMTkBarrierSetAssembler::generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm) {
+  generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_pre_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_pre_call), 3);
+}
+
+void MMTkBarrierSetAssembler::generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm) {
+  generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_post_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3);
+}
+
+void MMTkBarrierSetAssembler::generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm) {
+  generate_c1_runtime_stub_general(sasm, "c1_object_reference_write_slow_stub", FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3);
 }
 
 #undef __
 
+// Generate code stubs
+
 #define __ ce->masm()->
 
-void MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call(LIR_Assembler* ce, MMTkC1BarrierStub* stub) {
-  // printf("xxx MMTkBarrierSetAssembler::generate_c1_write_barrier_stub_call\n");
-  // See also void G1BarrierSetAssembler::gen_post_barrier_stub(LIR_Assembler* ce, G1PostBarrierStub* stub)
-  MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*) BarrierSet::barrier_set()->barrier_set_c1();
+void MMTkBarrierSetAssembler::generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub) {
+  MMTkBarrierSetC1* bs = (MMTkBarrierSetC1*)BarrierSet::barrier_set()->barrier_set_c1();
+
   __ bind(*stub->entry());
-  assert(stub->src->is_register(), "Precondition");
-  assert(stub->slot->is_register(), "Precondition");
-  assert(stub->new_val->is_register(), "Precondition");
-  ce->store_parameter(stub->src->as_pointer_register(), 0);
-  ce->store_parameter(stub->slot->as_pointer_register(), 1);
-  ce->store_parameter(stub->new_val->as_pointer_register(), 2);
-  __ far_call(RuntimeAddress(bs->_write_barrier_c1_runtime_code_blob->code_begin()));
+  assert(stub->val->is_register(), "Precondition.");
+
+  Register val_reg = stub->val->as_register();
+
+  __ cbz(val_reg, *stub->continuation());
+  ce->store_parameter(stub->val->as_register(), 0);
+  __ far_call(RuntimeAddress(bs->load_reference_c1_runtime_code_blob()->code_begin()));
   __ b(*stub->continuation());
 }
 

openjdk/cpu/aarch64/mmtkBarrierSetAssembler_aarch64.hpp

@@ -5,18 +5,16 @@
 #include "gc/shared/barrierSetAssembler.hpp"
 
 class MMTkBarrierSetC1;
-class MMTkC1BarrierStub;
+class MMTkC1ReferenceLoadBarrierStub;
 class LIR_Assembler;
 class StubAssembler;
 
 class MMTkBarrierSetAssembler: public BarrierSetAssembler {
-  friend class MMTkBarrierSetC1;
-
 protected:
   /// Full pre-barrier
-  virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {}
+  virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const {}
   /// Full post-barrier
-  virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {}
+  virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const {}
 
   /// Barrier elision test
   virtual bool can_remove_barrier(DecoratorSet decorators, Register val, bool skip_const_null) const {
@@ -26,9 +24,6 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler {
     return !in_heap || (skip_const_null && val == noreg);
   }
 
-  /// Generate C1 write barrier slow-call assembly code
-  virtual void generate_c1_write_barrier_runtime_stub(StubAssembler* sasm) const;
-
 public:
   virtual void eden_allocate(MacroAssembler* masm,
     Register obj,                      // result: pointer to object after successful allocation
@@ -39,12 +34,24 @@ class MMTkBarrierSetAssembler: public BarrierSetAssembler {
     Label&   slow_case                // continuation point if fast allocation fails
   );
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) {
-    if (type == T_OBJECT || type == T_ARRAY) object_reference_write_pre(masm, decorators, dst, val, tmp1, tmp2);
+    if (type == T_OBJECT || type == T_ARRAY) object_reference_write_pre(masm, decorators, dst, val, tmp1, tmp2, tmp3);
     BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
-    if (type == T_OBJECT || type == T_ARRAY) object_reference_write_post(masm, decorators, dst, val, tmp1, tmp2);
+    if (type == T_OBJECT || type == T_ARRAY) object_reference_write_post(masm, decorators, dst, val, tmp1, tmp2, tmp3);
   }
 
-  /// Generate C1 write barrier slow-call stub
-  static void generate_c1_write_barrier_stub_call(LIR_Assembler* ce, MMTkC1BarrierStub* stub);
+  //////////////////// Assembler for C1 ////////////////////
+
+  // Generate runtime stubs for the "runtime code blobs" in MMTkBarrierSetC1
+private:
+  static void generate_c1_runtime_stub_general(StubAssembler* sasm, const char* name, address func, int argc);
+public:
+  static void generate_c1_load_reference_runtime_stub(StubAssembler* sasm);
+  static void generate_c1_object_reference_write_pre_runtime_stub(StubAssembler* sasm);
+  static void generate_c1_object_reference_write_post_runtime_stub(StubAssembler* sasm);
+  static void generate_c1_object_reference_write_slow_runtime_stub(StubAssembler* sasm);
+
+  // Generate slow-path code stubs
+public:
+  static void generate_c1_ref_load_barrier_stub_call(LIR_Assembler* ce, MMTkC1ReferenceLoadBarrierStub* stub);
 };
 #endif // MMTK_OPENJDK_MMTK_BARRIER_SET_ASSEMBLER_AARCH64_HPP

openjdk/cpu/aarch64/mmtkNoBarrierSetAssembler_aarch64.hpp

@@ -1,5 +1,8 @@
 #ifndef MMTK_OPENJDK_MMTK_NO_BARRIER_SET_ASSEMBLER_AARCH64_HPP
 #define MMTK_OPENJDK_MMTK_NO_BARRIER_SET_ASSEMBLER_AARCH64_HPP
 
+#include "utilities/macros.hpp"
+#include CPU_HEADER(mmtkBarrierSetAssembler)
+
 class MMTkNoBarrierSetAssembler: public MMTkBarrierSetAssembler {};
 #endif // MMTK_OPENJDK_MMTK_NO_BARRIER_SET_ASSEMBLER_AARCH64_HPP

openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.cpp

@@ -2,52 +2,13 @@
 #include "mmtkObjectBarrier.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 
+//////////////////// Assembler ////////////////////
+
 #define __ masm->
 
-void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const {
-  // tmp1 and tmp2 is from MacroAssembler::access_store_at
-  // For do_oop_store, we have three tmps, x28/t3, x29/t4, x13/a3
-  // printf("object_reference_write_post\n");
-//   if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return;
+void MMTkObjectBarrierSetAssembler::object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const {
   if (can_remove_barrier(decorators, val, /* skip_const_null */ true)) return;
-  Register obj = dst.base();
-
-  if (mmtk_enable_barrier_fastpath) {
-    Label done;
-
-    assert_different_registers(obj, tmp1, tmp2);
-    assert_different_registers(val, tmp1, tmp2);
-    assert(tmp1->is_valid(), "need temp reg");
-    assert(tmp2->is_valid(), "need temp reg");
-    // tmp1 = load-byte (SIDE_METADATA_BASE_ADDRESS + (obj >> 6));
-    __ mov(tmp1, obj);
-    __ lsr(tmp1, tmp1, 6); // tmp1 = obj >> 6;
-    __ mov(tmp2, SIDE_METADATA_BASE_ADDRESS);
-    __ add(tmp1, tmp1, tmp2); // tmp1 = SIDE_METADATA_BASE_ADDRESS + (obj >> 6);
-    __ ldrb(tmp1, Address(tmp1, 0));
-    // tmp2 = (obj >> 3) & 7
-    __ mov(tmp2, obj);
-    __ lsr(tmp2, tmp2, 3);
-    __ andr(tmp2, tmp2, 7);
-    // tmp1 = tmp1 >> tmp2
-    __ lsrv(tmp1, tmp1, tmp2);
-    // if ((tmp1 & 1) == 1) fall through to slowpath;
-    // equivalently ((tmp1 & 1) == 0) go to done
-    __ andr(tmp1, tmp1, 1);
-    __ cbz(tmp1, done);
-    // setup calling convention
-    __ mov(c_rarg0, obj);
-    __ lea(c_rarg1, dst);
-    __ mov(c_rarg2, val == noreg ? zr : val);
-    __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_slow_call), 3);
-
-    __ bind(done);
-  } else {
-    __ mov(c_rarg0, obj);
-    __ lea(c_rarg1, dst);
-    __ mov(c_rarg2, val == noreg ? zr : val);
-    __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_write_post_call), 3);
-  }
+  object_reference_write_pre_or_post(masm, decorators, dst, val, tmp1, tmp2, tmp3, /* pre = */ false);
 }
 
 void MMTkObjectBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,

openjdk/cpu/aarch64/mmtkObjectBarrierSetAssembler_aarch64.hpp

@@ -1,11 +1,17 @@
 #ifndef MMTK_OPENJDK_MMTK_OBJECT_BARRIER_SET_ASSEMBLER_AARCH64_HPP
 #define MMTK_OPENJDK_MMTK_OBJECT_BARRIER_SET_ASSEMBLER_AARCH64_HPP
 
-class MMTkObjectBarrierSetAssembler: public MMTkBarrierSetAssembler {
+#include "utilities/macros.hpp"
+#include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler)
+
+//////////////////// Assembler ////////////////////
+
+class MMTkObjectBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler {
 protected:
-  virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2) const override;
+  virtual void object_reference_write_post(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const override;
 public:
   virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
                                   Register src, Register dst, Register count, Register tmp, RegSet saved_regs) override;
 };
+
 #endif // MMTK_OPENJDK_MMTK_OBJECT_BARRIER_SET_ASSEMBLER_AARCH64_HPP

openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.cpp

@@ -0,0 +1,63 @@
+#include "precompiled.hpp"
+#include "mmtkSATBBarrier.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+
+//////////////////// Assembler ////////////////////
+
+#define __ masm->
+
+void MMTkSATBBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread) {
+  bool on_oop = type == T_OBJECT || type == T_ARRAY;
+  bool on_weak = (decorators & ON_WEAK_OOP_REF) != 0;
+  bool on_phantom = (decorators & ON_PHANTOM_OOP_REF) != 0;
+  bool on_reference = on_weak || on_phantom;
+
+  BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+
+  if (mmtk_enable_reference_load_barrier) {
+    if (on_oop && on_reference) {
+      Label done;
+
+      assert_different_registers(dst, tmp1);
+
+      // No slow-call if SATB is not active
+      // intptr_t tmp1_q = CONCURRENT_MARKING_ACTIVE;
+      __ movptr(tmp1, intptr_t(&CONCURRENT_MARKING_ACTIVE));
+      // Load with zero extension to 32 bits.
+      // uint32_t tmp1_l = (uint32_t)(*(unt8_t*)tmp1_q);
+      __ ldrb(tmp1, Address(tmp1, 0));
+      // if (tmp1_l == 0) goto done;
+      __ cbz(tmp1, done);
+      // if (dst == 0) goto done;
+      __ cbz(dst, done);
+      // Do slow-call
+      __ push_call_clobbered_registers();
+      __ mov(c_rarg0, dst);
+      __ MacroAssembler::call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::load_reference_call), 1);
+      __ pop_call_clobbered_registers();
+      __ bind(done);
+    }
+  }
+}
+
+void MMTkSATBBarrierSetAssembler::object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const {
+  if (can_remove_barrier(decorators, val, /* skip_const_null */ false)) return;
+  object_reference_write_pre_or_post(masm, decorators, dst, val, tmp1, tmp2, tmp3, /* pre = */ true);
+}
+
+void MMTkSATBBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) {
+  if (is_oop) {
+    Label done;
+    // Skip the runtime call if count is zero.
+    __ cbz(count, done);
+    __ push_call_clobbered_registers();
+    __ mov(c_rarg0, src);
+    __ mov(c_rarg1, dst);
+    __ mov(c_rarg2, count);
+    __ call_VM_leaf(FN_ADDR(MMTkBarrierSetRuntime::object_reference_array_copy_pre_call), 3);
+    __ pop_call_clobbered_registers();
+    __ bind(done);
+  }
+}
+
+#undef __

openjdk/cpu/aarch64/mmtkSATBBarrierSetAssembler_aarch64.hpp

@@ -0,0 +1,17 @@
+#ifndef MMTK_OPENJDK_MMTK_SATB_BARRIER_SET_ASSEMBLER_AARCH64_HPP
+#define MMTK_OPENJDK_MMTK_SATB_BARRIER_SET_ASSEMBLER_AARCH64_HPP
+
+#include "utilities/macros.hpp"
+#include CPU_HEADER(mmtkUnlogBitBarrierSetAssembler)
+
+//////////////////// Assembler ////////////////////
+
+class MMTkSATBBarrierSetAssembler: public MMTkUnlogBitBarrierSetAssembler {
+protected:
+  virtual void object_reference_write_pre(MacroAssembler* masm, DecoratorSet decorators, Address dst, Register val, Register tmp1, Register tmp2, Register tmp3) const override;
+public:
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs) override;
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp2) override;
+};
+
+#endif // MMTK_OPENJDK_MMTK_SATB_BARRIER_SET_ASSEMBLER_AARCH64_HPP