Adapt JDK-8340796: Use a consistent order when loading cxq and EntryList

Author: mur47x111

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/amd64/z/AMD64HotSpotZBarrierSetLIRGenerator.java

@@ -137,7 +137,7 @@ public static void zUncolor(CompilationResultBuilder crb, AMD64MacroAssembler ma
      * slow path call to the runtime.
      */
     @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L304-L321", sha1 = "9a628c1771df79ae8b4cee89d2863fbd4a4964bc")
-    @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L370-L414", sha1 = "7688e7aeab5f1aa413690066355a17c18a4273fa")
+    @SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L374-L418", sha1 = "7688e7aeab5f1aa413690066355a17c18a4273fa")
     public static void emitPreWriteBarrier(CompilationResultBuilder crb,
                     AMD64MacroAssembler masm,
                     LIRInstruction op,
@@ -213,7 +213,7 @@ public static void emitPreWriteBarrier(CompilationResultBuilder crb,
     /**
      * Try to perform any local store barrier fixups or dispatch to the slow path.
      */
-    @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L450-L505", sha1 = "4b729acf92e6a297229b7f1e957601708c315f4f")
+    @SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L454-L509", sha1 = "4b729acf92e6a297229b7f1e957601708c315f4f")
     static void storeBarrierMedium(CompilationResultBuilder crb,
                     AMD64MacroAssembler masm,
                     AMD64Address address,
@@ -278,7 +278,7 @@ static void storeBarrierMedium(CompilationResultBuilder crb,
     /**
      * Add a value to the store buffer.
      */
-    @SyncPort(from = "https://github.com/openjdk/jdk/blob/4acafb809c66589fbbfee9c9a4ba7820f848f0e4/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L416-L448", sha1 = "638b10c65bb14fa4b254efa4d5bbb1751fdbb6bf")
+    @SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp#L420-L452", sha1 = "638b10c65bb14fa4b254efa4d5bbb1751fdbb6bf")
     static void storeBarrierBufferAdd(AMD64MacroAssembler masm,
                     AMD64Address address,
                     Register tmp1,

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/hotspot/replacements/MonitorSnippets.java

@@ -189,8 +189,8 @@
  * appropriately to comply with the layouts above.
  */
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/180affc5718c9bf2f009d6a7aa129cc36335384a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L175-L532",
-          sha1 = "6269b3e841e4a6be49042270f3489eae292fc05f")
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L175-L532",
+          sha1 = "042c4a5c1730525a53c24ee194a182dfd34bf754")
 // @formatter:on
 public class MonitorSnippets implements Snippets {
 
@@ -515,8 +515,8 @@ private static boolean tryStackUnlocking(Object object, Word thread, Word lock,
     }
 
     // @formatter:off
-    @SyncPort(from = "https://github.com/openjdk/jdk/blob/dcac4b0a532f2ca6cb374da7ece331e8266ab351/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L694-L855",
-              sha1 = "410864d3b17fd0e3f1026b9c06e4d383446896ba")
+    @SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L694-L858",
+              sha1 = "57536f7432c20e2e785059b464948f9aeee3804b")
     // @formatter:on
     private static boolean tryLightweightUnlocking(Object object, Word thread, Word lock, boolean trace, Counters counters) {
         // Load top
@@ -597,12 +597,10 @@ private static boolean tryExitInflated(Object object, Word thread, Word lock, bo
         if (probability(FAST_PATH_PROBABILITY, recursions.equal(0))) {
             if (JavaVersionUtil.JAVA_SPEC == 21) {
                 // recursions == 0
-                int cxqOffset = objectMonitorCxqOffset(INJECTED_VMCONFIG);
-                Word cxq = monitor.readWord(cxqOffset, OBJECT_MONITOR_CXQ_LOCATION);
-                int entryListOffset = objectMonitorEntryListOffset(INJECTED_VMCONFIG);
-                Word entryList = monitor.readWord(entryListOffset, OBJECT_MONITOR_ENTRY_LIST_LOCATION);
-                if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
-                    // cxq == 0 && entryList == 0
+                Word entryList = monitor.readWord(objectMonitorEntryListOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_ENTRY_LIST_LOCATION);
+                Word cxq = monitor.readWord(objectMonitorCxqOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_CXQ_LOCATION);
+                if (probability(FREQUENT_PROBABILITY, entryList.or(cxq).equal(0))) {
+                    // entryList == 0 && cxq == 0
                     // Nobody is waiting, success
                     // release_store
                     memoryBarrier(MembarNode.FenceKind.STORE_RELEASE);
@@ -641,10 +639,17 @@ private static boolean tryExitInflated(Object object, Word thread, Word lock, bo
                 memoryBarrier(MembarNode.FenceKind.STORE_RELEASE);
                 monitor.writeWord(ownerOffset, zero());
                 memoryBarrier(MembarNode.FenceKind.STORE_LOAD);
-                Word cxq = monitor.readWord(objectMonitorCxqOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_CXQ_LOCATION);
+                // Note that we read the EntryList and then the cxq after dropping the
+                // lock, so the values need not form a stable snapshot. In particular,
+                // after reading the (empty) EntryList, another thread could acquire
+                // and release the lock, moving any entries in the cxq to the
+                // EntryList, causing the current thread to see an empty cxq and
+                // conclude there are no waiters. But this is okay as the thread that
+                // moved the cxq is responsible for waking the successor.
                 Word entryList = monitor.readWord(objectMonitorEntryListOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_ENTRY_LIST_LOCATION);
+                Word cxq = monitor.readWord(objectMonitorCxqOffset(INJECTED_VMCONFIG), OBJECT_MONITOR_CXQ_LOCATION);
                 // Check if the entry lists are empty.
-                if (probability(FREQUENT_PROBABILITY, cxq.or(entryList).equal(0))) {
+                if (probability(FREQUENT_PROBABILITY, entryList.or(cxq).equal(0))) {
                     traceObject(trace, "-lock{heavyweight:simple}", object, false);
                     counters.unlockHeavySimple.inc();
                     return true;

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64CountPositivesOp.java

@@ -69,7 +69,7 @@
  * Returns the number of positive bytes.
  */
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/dcac4b0a532f2ca6cb374da7ece331e8266ab351/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L5484-L5553",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L5467-L5536",
           sha1 = "ce54a7cf2fcfe7ccb8f6604c038887fc1c4ebce1")
 @SyncPort(from = "https://github.com/openjdk/jdk/blob/7131f053b0d26b62cbf0d8376ec117d6e8d79f9e/src/hotspot/cpu/aarch64/stubGenerator_aarch64.cpp#L4975-L5141",
           sha1 = "3b4e6edb4372e8babb009763c2d05961348dd723")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64EncodeArrayOp.java

@@ -51,7 +51,7 @@
 import jdk.vm.ci.meta.Value;
 
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/dcac4b0a532f2ca6cb374da7ece331e8266ab351/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6087-L6200",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6070-L6183",
           sha1 = "80e6323172af5e8a33625b4eb14629cdad500a0f")
 // @formatter:on
 @Opcode("AArch64_ENCODE_ARRAY")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64RoundFloatToIntegerOp.java

@@ -45,7 +45,7 @@
  * {@link Math#round} algorithm for details.
  */
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/dcac4b0a532f2ca6cb374da7ece331e8266ab351/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6320-L6368",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6303-L6351",
           sha1 = "76d47473bf8d1408bf6e7bf6b8a3d93c19dab9c6")
 // @formatter:on
 @Opcode("AARCH64_ROUND_FLOAT_TO_INTEGER")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/aarch64/AArch64StringUTF16CompressOp.java

@@ -44,7 +44,7 @@
 import jdk.vm.ci.meta.Value;
 
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/dcac4b0a532f2ca6cb374da7ece331e8266ab351/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6308-L6318",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp#L6291-L6301",
           sha1 = "857dc6f9a492da6c8e20afb2139ae393efd228ac")
 // @formatter:on
 @Opcode("AArch64_STRING_COMPRESS")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64BitSwapOp.java

@@ -40,7 +40,7 @@
 import jdk.vm.ci.meta.Value;
 
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L6256-L6338",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L6457-L6539",
           sha1 = "34c6e1ee7916fc7190cbcbc237eaf2b510f7dd0e")
 // @formatter:on
 public final class AMD64BitSwapOp extends AMD64LIRInstruction {

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64CountPositivesOp.java

@@ -55,7 +55,7 @@
 import jdk.vm.ci.meta.Value;
 
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L4012-L4282",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L4084-L4354",
           sha1 = "684b5353c58bbf92e4403aa985113a78a1f38930")
 // @formatter:on
 @Opcode("AMD64_COUNT_POSITIVES")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64RoundFloatToIntegerOp.java

@@ -46,7 +46,7 @@
  * {@link Math#round} algorithm for details.
  */
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7131f053b0d26b62cbf0d8376ec117d6e8d79f9e/src/hotspot/cpu/x86/macroAssembler_x86.cpp#L10054-L10150",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/macroAssembler_x86.cpp#L10138-L10234",
           sha1 = "9e13c7375bbb35809ad79ebd6a9cc19e66f57aa1")
 @SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp#L596-L763",
           sha1 = "312f16a0551887f78cc567638477bbbcbc3765c5")

compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/lir/amd64/AMD64VectorizedHashCodeOp.java

@@ -75,11 +75,11 @@
 import jdk.vm.ci.meta.Value;
 
 // @formatter:off
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L1769-L1878",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L1838-L1947",
           sha1 = "1cc5a10b19e7746105493d8f430f628cc7f89c51")
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L2069-L2115",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L2138-L2184",
           sha1 = "9cbba8bd6c4037427fa46f067abb722b15aca90c")
-@SyncPort(from = "https://github.com/openjdk/jdk/blob/7fa2f229fbee68112cbdd18b811d95721adfe2ec/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L3427-L3614",
+@SyncPort(from = "https://github.com/openjdk/jdk/blob/d8430efb5e159b8e08d2cac66b46cb4ff1112927/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp#L3499-L3686",
           sha1 = "2457cf3f9d3ff89c1515fa5d95cc7c8437a5318b")
 // @formatter:on
 @Opcode("VECTORIZED_HASHCODE")