fix old object constant bug

Author: roberttoyonaga

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/UninterruptiblePriorityQueue.java

@@ -51,6 +51,11 @@ public boolean isFull() {
         return size == queue.length;
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public boolean isEmpty() {
+        return size == 0;
+    }
+
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public boolean add(UninterruptibleComparable e) {
         return offer(e);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java

@@ -52,6 +52,7 @@
 import com.oracle.svm.core.locks.VMMutex;
 import com.oracle.svm.core.memory.NullableNativeMemory;
 import com.oracle.svm.core.nmt.NmtCategory;
+import com.oracle.svm.core.sampler.SamplerBuffer;
 import com.oracle.svm.core.sampler.SamplerSampleWriter;
 import com.oracle.svm.core.sampler.SamplerSampleWriterData;
 import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess;
@@ -135,6 +136,49 @@ public long getStackTraceId(int skipCount) {
         }
     }
 
+    /** Don't store to the stacktrace repo here (it may not belong in this epoch). */
+    @NeverInline("Starting a stack walk in the caller frame.")
+    @Uninterruptible(reason = "Result is only valid until epoch changes.", callerMustBe = true)
+    public boolean recordStackTraceDataToBuffer(int skipCount, SamplerBuffer rawStackTraceData) {
+        if (DeoptimizationSupport.enabled() || rawStackTraceData.isNull()) {
+            /* Stack traces are not supported if JIT compilation is used (GR-43686). */
+            return false;
+        }
+
+        /**
+         * The JFR sampler does not share the same buffers as the leak profiler. There is no risk of
+         * races between them.
+         */
+        SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class);
+        if (rawStackTraceData.isNull() || !SamplerSampleWriterDataAccess.initialize(data, skipCount, true, rawStackTraceData)) {
+            return false;
+        }
+
+        assert SamplerSampleWriterDataAccess.verify(data);
+        assert data.getCurrentPos().unsignedRemainder(Long.BYTES).equal(0);
+
+        /*
+         * Start a stack trace and do a stack walk. Note that the data will only be committed to the
+         * buffer if it is a new stack trace.
+         */
+        SamplerSampleWriter.begin(data);
+        Pointer sp = KnownIntrinsics.readCallerStackPointer();
+        CodePointer ip = FrameAccess.singleton().readReturnAddress(CurrentIsolate.getCurrentThread(), sp);
+        int errorCode = JfrStackWalker.walkCurrentThread(data, ip, sp, false);
+
+        /*
+         * If writing the raw data to the buffer succeeded, commit it here. Do not attempt
+         * deduplication yet since the event data this stack trace corresponds to may not be emitted
+         * until a much later epoch.
+         */
+        return switch (errorCode) {
+            case JfrStackWalker.NO_ERROR, JfrStackWalker.TRUNCATED -> SamplerSampleWriter.end(data, SamplerSampleWriter.JFR_STACK_TRACE_END);
+            case JfrStackWalker.BUFFER_SIZE_EXCEEDED -> false;
+            case JfrStackWalker.UNPARSEABLE_STACK -> throw VMError.shouldNotReachHere("Only the async sampler may encounter an unparseable stack.");
+            default -> throw VMError.shouldNotReachHere("Unexpected return value");
+        };
+    }
+
     @Uninterruptible(reason = "Result is only valid until epoch changes.", callerMustBe = true)
     private long storeDeduplicatedStackTrace(SamplerSampleWriterData data) {
         if (SamplerSampleWriter.isValid(data)) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java

@@ -43,6 +43,7 @@
 import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler;
 import com.oracle.svm.core.jfr.throttling.JfrEventThrottling;
 import com.oracle.svm.core.log.Log;
+import com.oracle.svm.core.sampler.SamplerBuffer;
 import com.oracle.svm.core.sampler.SamplerBufferPool;
 import com.oracle.svm.core.sampler.SamplerBuffersAccess;
 import com.oracle.svm.core.sampler.SamplerStatistics;
@@ -281,6 +282,18 @@ public long getStackTraceId(long eventTypeId, int skipCount) {
         }
     }
 
+    /*
+     * Buffers are not shared with the sampler or other JFR event types. This data will not yet be
+     * tied to a specific epoch until event emission.
+     */
+    @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public boolean recordStackTraceDataToBuffer(JfrEvent eventType, int skipCount, SamplerBuffer rawStackTraceData) {
+        if (isRecording() && isStackTraceEnabled(eventType.getId())) {
+            return stackTraceRepo.recordStackTraceDataToBuffer(skipCount, rawStackTraceData);
+        }
+        return false;
+    }
+
     /**
      * See {@link JVM#getStackTraceId}.
      */

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObject.java

@@ -32,12 +32,18 @@
 
 import jdk.graal.compiler.word.Word;
 import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.collections.UninterruptibleComparable;
 import com.oracle.svm.core.collections.UninterruptibleLinkedList;
 import com.oracle.svm.core.heap.ReferenceInternals;
 import com.oracle.svm.core.jfr.JfrTicks;
+import com.oracle.svm.core.memory.NullableNativeMemory;
+import com.oracle.svm.core.nmt.NmtCategory;
+import com.oracle.svm.core.sampler.SamplerBuffer;
+import com.oracle.svm.core.sampler.SamplerBufferAccess;
+import com.oracle.svm.core.jfr.SubstrateJVM;
 
 /**
  * Holds information about a sampled object. This data may only be accessed while holding the
@@ -50,26 +56,29 @@ public final class JfrOldObject implements UninterruptibleComparable, Uninterrup
     private UnsignedWord span;
     private UnsignedWord objectSize;
     private long allocationTicks;
-    private long threadId;
-    private long stackTraceId;
     private UnsignedWord heapUsedAfterLastGC;
     private int arrayLength;
+    private SamplerBuffer rawStackTraceData;
+    private Thread thread;
+    private boolean stackTraceValid;
 
     JfrOldObject() {
         this.reference = new WeakReference<>(null);
+        this.rawStackTraceData = WordFactory.nullPointer();
     }
 
     @SuppressWarnings("hiding")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    void initialize(Object obj, UnsignedWord span, UnsignedWord allocatedSize, long threadId, long stackTraceId, UnsignedWord heapUsedAfterLastGC, int arrayLength) {
+    void initialize(Object obj, UnsignedWord span, UnsignedWord allocatedSize, Thread thread, UnsignedWord heapUsedAfterLastGC, int arrayLength) {
         ReferenceInternals.setReferent(reference, obj);
         this.span = span;
         this.objectSize = allocatedSize;
         this.allocationTicks = JfrTicks.elapsedTicks();
-        this.threadId = threadId;
-        this.stackTraceId = stackTraceId;
         this.heapUsedAfterLastGC = heapUsedAfterLastGC;
         this.arrayLength = arrayLength;
+        this.thread = thread;
+        this.stackTraceValid = false;
+        this.rawStackTraceData = WordFactory.nullPointer();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
@@ -78,11 +87,12 @@ void reset() {
         this.span = Word.zero();
         this.objectSize = Word.zero();
         this.allocationTicks = 0L;
-        this.threadId = 0L;
-        this.stackTraceId = 0L;
         this.heapUsedAfterLastGC = Word.zero();
         this.arrayLength = 0;
         this.next = null;
+        this.thread = null;
+        this.stackTraceValid = false;
+        freeRawStackTraceData();
     }
 
     @Override
@@ -118,23 +128,53 @@ public long getAllocationTicks() {
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public long getThreadId() {
-        return threadId;
+    public UnsignedWord getHeapUsedAfterLastGC() {
+        return heapUsedAfterLastGC;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public long getStackTraceId() {
-        return stackTraceId;
+    public int getArrayLength() {
+        return arrayLength;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public UnsignedWord getHeapUsedAfterLastGC() {
-        return heapUsedAfterLastGC;
+    public Thread getThread() {
+        return thread;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public int getArrayLength() {
-        return arrayLength;
+    public boolean getStackTraceValid() {
+        return stackTraceValid;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public void setStackTraceValid(boolean valid) {
+        stackTraceValid = valid;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public SamplerBuffer getRawStackTraceData() {
+        if (rawStackTraceData.isNull()) {
+            // TODO add OldObjectSample event skipcount here too.
+            long bufferDataSize = Long.SIZE * SubstrateJVM.getStackTraceRepo().getStackTraceDepth();
+            rawStackTraceData = NullableNativeMemory.malloc(SamplerBufferAccess.getTotalBufferSize(WordFactory.unsigned(bufferDataSize)), NmtCategory.JFR);
+            if (rawStackTraceData.isNonNull()) {
+                rawStackTraceData.setSize(WordFactory.unsigned(bufferDataSize));
+                rawStackTraceData.setNext(Word.nullPointer());
+                SamplerBufferAccess.reinitialize(rawStackTraceData);
+            } else {
+                this.stackTraceValid = false;
+            }
+        }
+        return rawStackTraceData;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    private void freeRawStackTraceData() {
+        if (rawStackTraceData.isNonNull()) {
+            NullableNativeMemory.free(rawStackTraceData);
+            this.rawStackTraceData = WordFactory.nullPointer();
+        }
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObjectProfiler.java

@@ -64,10 +64,15 @@ public void configure(int oldObjectQueueSize) {
     }
 
     public void reset() {
+        if (this.sampler != null) {
+            this.sampler.reset();
+        }
         this.sampler = new JfrOldObjectSampler(queueSize);
     }
 
     public void teardown() {
+        // Reset to free buffers.
+        this.sampler.reset();
         this.sampler = null;
     }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObjectSampler.java

@@ -28,7 +28,9 @@
 
 import static com.oracle.svm.core.Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE;
 
+import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.util.BasedOnJDKFile;
 import com.oracle.svm.core.Uninterruptible;
@@ -39,6 +41,10 @@
 import com.oracle.svm.core.jfr.JfrTicks;
 import com.oracle.svm.core.jfr.SubstrateJVM;
 import com.oracle.svm.core.jfr.events.OldObjectSampleEvent;
+import com.oracle.svm.core.sampler.SamplerBuffer;
+import com.oracle.svm.core.sampler.SamplerBufferAccess;
+import com.oracle.svm.core.sampler.SamplerJfrStackTraceSerializer;
+import com.oracle.svm.core.sampler.SamplerSampleWriter;
 import com.oracle.svm.core.thread.JavaThreads;
 
 /**
@@ -145,20 +151,21 @@ private void evict() {
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private void release(JfrOldObject sample) {
         usedList.remove(sample);
-        freeList.append(sample);
         sample.reset();
+        freeList.append(sample);
     }
 
     @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+26/src/hotspot/share/jfr/leakprofiler/sampling/samplePriorityQueue.cpp#L44-L53")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private void store(Object obj, UnsignedWord span, UnsignedWord allocatedSize, int arrayLength) {
         Thread thread = JavaThreads.getCurrentThreadOrNull();
-        long threadId = thread == null ? 0L : JavaThreads.getThreadId(thread);
-        long stackTraceId = thread == null ? 0L : SubstrateJVM.get().getStackTraceId(JfrEvent.OldObjectSample, 0);
         UnsignedWord heapUsedAfterLastGC = Heap.getHeap().getUsedMemoryAfterLastGC();
 
         JfrOldObject sample = (JfrOldObject) freeList.pop();
-        sample.initialize(obj, span, allocatedSize, threadId, stackTraceId, heapUsedAfterLastGC, arrayLength);
+        sample.initialize(obj, span, allocatedSize, thread, heapUsedAfterLastGC, arrayLength);
+        // Allocate a raw stacktrace buffer and write to it.
+        boolean stackTraceValid = SubstrateJVM.get().recordStackTraceDataToBuffer(JfrEvent.OldObjectSample, 0, sample.getRawStackTraceData());
+        sample.setStackTraceValid(stackTraceValid);
         queue.add(sample);
         usedList.append(sample);
         totalInQueue = totalInQueue.add(span);
@@ -184,8 +191,16 @@ private void emitUnchained() {
                 long objectId = SubstrateJVM.getOldObjectRepository().serializeOldObject(obj);
                 UnsignedWord objectSize = cur.getObjectSize();
                 long allocationTicks = cur.getAllocationTicks();
-                long threadId = cur.getThreadId();
-                long stackTraceId = cur.getStackTraceId();
+                Thread thread = cur.getThread();
+                long threadId = 0L;
+                if (thread != null) {
+                    threadId = JavaThreads.getThreadId(thread);
+                    SubstrateJVM.getThreadRepo().registerThread(thread);
+                }
+                long stackTraceId = 0L;
+                if (cur.getStackTraceValid()) {
+                    stackTraceId = deduplicateAndSerializeStackTrace(cur.getRawStackTraceData());
+                }
                 UnsignedWord heapUsedAfterLastGC = cur.getHeapUsedAfterLastGC();
                 int arrayLength = cur.getArrayLength();
 
@@ -196,6 +211,39 @@ private void emitUnchained() {
         }
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    private static long deduplicateAndSerializeStackTrace(SamplerBuffer buffer) {
+        if (buffer.isNull()) {
+            // Buffer may have failed allocation.
+            return 0L;
+        }
+
+        // decode the buffer
+        Pointer bufferEnd = buffer.getPos();
+        Pointer current = SamplerBufferAccess.getDataStart(buffer);
+        Pointer entryStart = current;
+        assert entryStart.unsignedRemainder(Long.BYTES).equal(0);
+
+        /* Sample hash. */
+        int sampleHash = current.readInt(0);
+        current = current.add(Integer.BYTES);
+
+        /* Is truncated. */
+        boolean isTruncated = current.readInt(0) == 1;
+        current = current.add(Integer.BYTES);
+
+        /* Sample size, excluding the header and the end marker. */
+        int sampleSize = current.readInt(0);
+        current = current.add(Integer.BYTES);
+
+        /* skip over padding, ticks, event thread, and thread state. */
+        current = current.add(Integer.BYTES).add(Long.BYTES * 3);
+
+        assert current.subtract(entryStart).equal(SamplerSampleWriter.getHeaderSize());
+
+        return SamplerJfrStackTraceSerializer.deduplicateAndSerializeStackTrace(current, bufferEnd, sampleSize, sampleHash, isTruncated);
+    }
+
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     JfrOldObject getOldestObject() {
         return (JfrOldObject) usedList.getHead();
@@ -205,4 +253,14 @@ JfrOldObject getOldestObject() {
     private JfrOldObject getObjectWithSmallestSpan() {
         return (JfrOldObject) queue.peek();
     }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public void reset() {
+        while (!queue.isEmpty()) {
+            evict();
+        }
+        assert usedList.isEmpty();
+        totalAllocated = WordFactory.unsigned(0);
+        totalInQueue = WordFactory.unsigned(0);
+    }
 }