[GR-60075] Allow using different LocationIdentities with ObjectHeader.initializeObjectHeader

c-refice · c-refice · commit 73282d73d16a · 2024-11-30T03:39:21.000Z
PullRequest: graal/19444
diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,6 @@
 
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
-import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordBase;
@@ -48,7 +47,6 @@
 
 import jdk.graal.compiler.api.directives.GraalDirectives;
 import jdk.graal.compiler.api.replacements.Fold;
-import jdk.graal.compiler.nodes.extended.MembarNode;
 import jdk.graal.compiler.replacements.ReplacementsUtil;
 import jdk.graal.compiler.word.ObjectAccess;
 import jdk.graal.compiler.word.Word;
@@ -137,26 +135,26 @@ public Word encodeAsUnmanagedObjectHeader(DynamicHub hub) {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
     @Override
-    public void initializeHeaderOfNewObject(Pointer objectPointer, Word encodedHub, boolean isArrayLike) {
+    protected void initializeObjectHeader(Pointer objectPointer, Word encodedHub, boolean isArrayLike, MemWriter writer) {
         ObjectLayout ol = ConfigurationValues.getObjectLayout();
         boolean isIdentityHashFieldInObjectHeader = ol.isIdentityHashFieldInObjectHeader() || ol.isIdentityHashFieldAtTypeSpecificOffset() && isArrayLike;
         if (getReferenceSize() == Integer.BYTES) {
             dynamicAssert(encodedHub.and(WordFactory.unsigned(0xFFFFFFFF00000000L)).isNull(), "hub can only use 32 bits");
             if (isIdentityHashFieldInObjectHeader) {
                 /* Use a single 64-bit write to initialize the hub and the identity hashcode. */
                 dynamicAssert(ol.getObjectHeaderIdentityHashOffset() == getHubOffset() + 4, "assumed layout to optimize initializing write");
-                objectPointer.writeLong(getHubOffset(), encodedHub.rawValue(), LocationIdentity.INIT_LOCATION);
+                writer.writeLong(objectPointer, getHubOffset(), encodedHub.rawValue());
             } else {
-                objectPointer.writeInt(getHubOffset(), (int) encodedHub.rawValue(), LocationIdentity.INIT_LOCATION);
+                writer.writeInt(objectPointer, getHubOffset(), (int) encodedHub.rawValue());
             }
         } else {
-            objectPointer.writeWord(getHubOffset(), encodedHub, LocationIdentity.INIT_LOCATION);
+            writer.writeWord(objectPointer, getHubOffset(), encodedHub);
             if (isIdentityHashFieldInObjectHeader) {
-                objectPointer.writeInt(ol.getObjectHeaderIdentityHashOffset(), 0, LocationIdentity.INIT_LOCATION);
+                writer.writeInt(objectPointer, ol.getObjectHeaderIdentityHashOffset(), 0);
             }
         }
-        MembarNode.memoryBarrier(MembarNode.FenceKind.ALLOCATION_INIT, LocationIdentity.INIT_LOCATION);
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,6 @@
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.type.CTypeConversion;
-import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.PointerBase;
 import org.graalvm.word.UnsignedWord;
@@ -55,7 +54,7 @@
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 import com.oracle.svm.core.util.VMError;
 
-import jdk.graal.compiler.nodes.extended.MembarNode;
+import jdk.graal.compiler.nodes.NamedLocationIdentity;
 import jdk.graal.compiler.nodes.java.ArrayLengthNode;
 import jdk.graal.compiler.word.Word;
 
@@ -102,9 +101,8 @@ private static <T extends NonmovableArray<?>> T createArray(int length, Class<?>
 
         ObjectHeader header = Heap.getHeap().getObjectHeader();
         Word encodedHeader = header.encodeAsUnmanagedObjectHeader(hub);
-        header.initializeHeaderOfNewObject(array, encodedHeader, true);
-        array.writeInt(ConfigurationValues.getObjectLayout().getArrayLengthOffset(), length, LocationIdentity.INIT_LOCATION);
-        MembarNode.memoryBarrier(MembarNode.FenceKind.ALLOCATION_INIT, LocationIdentity.INIT_LOCATION);
+        header.initializeHeaderOfNewObjectOffHeap(array, encodedHeader, true);
+        array.writeInt(ConfigurationValues.getObjectLayout().getArrayLengthOffset(), length, NamedLocationIdentity.OFF_HEAP_LOCATION);
         // already zero-initialized thanks to calloc()
         trackUnmanagedArray((NonmovableArray<?>) array);
         return (T) array;
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/SubstrateAllocationSnippets.java
@@ -529,7 +529,7 @@ protected final Object callNewMultiArrayStub(Word objectHeader, int rank, Word d
 
     @Override
     public void initializeObjectHeader(Word memory, Word objectHeader, boolean isArrayLike) {
-        Heap.getHeap().getObjectHeader().initializeHeaderOfNewObject(memory, objectHeader, isArrayLike);
+        Heap.getHeap().getObjectHeader().initializeHeaderOfNewObjectInit(memory, objectHeader, isArrayLike);
     }
 
     @Override
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,18 +26,22 @@
 
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
+import org.graalvm.word.LocationIdentity;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.WordFactory;
 
+import com.oracle.svm.core.AlwaysInline;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.config.ConfigurationValues;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.image.ImageHeapObject;
 import com.oracle.svm.core.snippets.KnownIntrinsics;
 
 import jdk.graal.compiler.api.replacements.Fold;
+import jdk.graal.compiler.nodes.NamedLocationIdentity;
 import jdk.graal.compiler.word.ObjectAccess;
 import jdk.graal.compiler.word.Word;
+import jdk.graal.compiler.word.WordOperationPlugin;
 
 /**
  * An object header is a reference-sized collection of bits in each object instance. The object
@@ -49,6 +53,8 @@
  * of this instance if the object has been moved by the collector.
  */
 public abstract class ObjectHeader {
+    protected static final String INLINE_INITIALIZE_HEADER_INIT_REASON = "Methods that write to INIT_LOCATION must be inlined into a caller that emits an ALLOCATION_INIT barrier.";
+
     @Platforms(Platform.HOSTED_ONLY.class)
     protected ObjectHeader() {
     }
@@ -107,8 +113,37 @@ public Pointer readPotentialDynamicHubFromPointer(Pointer ptr) {
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public abstract Pointer extractPotentialDynamicHubFromHeader(Word header);
 
+    /**
+     * Initializes the header of a newly allocated heap object (i.e. writing to
+     * {@link LocationIdentity#INIT_LOCATION})
+     */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
+    public final void initializeHeaderOfNewObjectInit(Pointer ptr, Word header, boolean isArrayLike) {
+        initializeObjectHeader(ptr, header, isArrayLike, InitLocationMemWriter.INSTANCE);
+    }
+
+    /**
+     * Initializes the header of a newly allocated object located anywhere in memory (i.e. writing
+     * to {@link LocationIdentity#ANY_LOCATION})
+     */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public abstract void initializeHeaderOfNewObject(Pointer ptr, Word header, boolean isArrayLike);
+    public final void initializeHeaderOfNewObjectAny(Pointer ptr, Word header, boolean isArrayLike) {
+        initializeObjectHeader(ptr, header, isArrayLike, AnyLocationMemWriter.INSTANCE);
+    }
+
+    /**
+     * Initializes the header of a newly allocated object located off the heap (i.e. writing to
+     * {@link NamedLocationIdentity#OFF_HEAP_LOCATION})
+     */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public final void initializeHeaderOfNewObjectOffHeap(Pointer ptr, Word header, boolean isArrayLike) {
+        initializeObjectHeader(ptr, header, isArrayLike, OffHeapLocationMemWriter.INSTANCE);
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
+    protected abstract void initializeObjectHeader(Pointer ptr, Word header, boolean isArrayLike, MemWriter writer);
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public boolean pointsToObjectHeader(Pointer ptr) {
@@ -149,4 +184,94 @@ protected static int getReferenceSize() {
     protected static int getHubOffset() {
         return ConfigurationValues.getObjectLayout().getHubOffset();
     }
+
+    /**
+     * Helper interface to write to memory at an overridable {@link LocationIdentity}.
+     * <p>
+     * {@link #initializeObjectHeader} is used in multiple contexts which write to different memory
+     * locations. LocationIdentity arguments to {@link WordOperationPlugin}s are required to be
+     * constant at the time of bytecode parsing, meaning it's not possible to {@link Fold} a
+     * location identity in any way, it must be a compile-time constant. To avoid duplicating
+     * implementations of {@link #initializeObjectHeader}, we get around this by delegating the
+     * writing to the MemWriter, whose implementations use different (constant) location identities.
+     */
+    protected interface MemWriter {
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        void writeWord(Pointer ptr, int offset, Word word);
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        void writeInt(Pointer ptr, int offset, int val);
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        void writeLong(Pointer ptr, int offset, long val);
+    }
+
+    private static final class AnyLocationMemWriter implements MemWriter {
+        private static final AnyLocationMemWriter INSTANCE = new AnyLocationMemWriter();
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeWord(Pointer ptr, int offset, Word word) {
+            ptr.writeWord(offset, word, LocationIdentity.ANY_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeInt(Pointer ptr, int offset, int val) {
+            ptr.writeInt(offset, val, LocationIdentity.ANY_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeLong(Pointer ptr, int offset, long val) {
+            ptr.writeLong(offset, val, LocationIdentity.ANY_LOCATION);
+        }
+    }
+
+    private static final class OffHeapLocationMemWriter implements MemWriter {
+        private static final OffHeapLocationMemWriter INSTANCE = new OffHeapLocationMemWriter();
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeWord(Pointer ptr, int offset, Word word) {
+            ptr.writeWord(offset, word, NamedLocationIdentity.OFF_HEAP_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeInt(Pointer ptr, int offset, int val) {
+            ptr.writeInt(offset, val, NamedLocationIdentity.OFF_HEAP_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public void writeLong(Pointer ptr, int offset, long val) {
+            ptr.writeLong(offset, val, NamedLocationIdentity.OFF_HEAP_LOCATION);
+        }
+    }
+
+    private static final class InitLocationMemWriter implements MemWriter {
+        private static final InitLocationMemWriter INSTANCE = new InitLocationMemWriter();
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
+        public void writeWord(Pointer ptr, int offset, Word word) {
+            ptr.writeWord(offset, word, LocationIdentity.INIT_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
+        public void writeInt(Pointer ptr, int offset, int val) {
+            ptr.writeInt(offset, val, LocationIdentity.INIT_LOCATION);
+        }
+
+        @Override
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        @AlwaysInline(INLINE_INITIALIZE_HEADER_INIT_REASON)
+        public void writeLong(Pointer ptr, int offset, long val) {
+            ptr.writeLong(offset, val, LocationIdentity.INIT_LOCATION);
+        }
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -529,7 +529,7 @@ protected final Object callNewMultiArrayStub(Word objectHeader, int rank, Word d`
`529`	`529`
`530`	`530`	`@Override`
`531`	`531`	`public void initializeObjectHeader(Word memory, Word objectHeader, boolean isArrayLike) {`
`532`		`- Heap.getHeap().getObjectHeader().initializeHeaderOfNewObject(memory, objectHeader, isArrayLike);`
	`532`	`+ Heap.getHeap().getObjectHeader().initializeHeaderOfNewObjectInit(memory, objectHeader, isArrayLike);`
`533`	`533`	`}`
`534`	`534`
`535`	`535`	`@Override`