Fix JFR leak profiler bug and annotate with BasedOnJDKFile

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/collections/UninterruptiblePriorityQueue.java

@@ -51,6 +51,11 @@ public boolean isFull() {
         return size == queue.length;
     }
 
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public boolean isEmpty() {
+        return size == 0;
+    }
+
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     public boolean add(UninterruptibleComparable e) {
         return offer(e);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/JfrStackTraceRepository.java

@@ -52,6 +52,7 @@
 import com.oracle.svm.core.locks.VMMutex;
 import com.oracle.svm.core.memory.NullableNativeMemory;
 import com.oracle.svm.core.nmt.NmtCategory;
+import com.oracle.svm.core.sampler.SamplerBuffer;
 import com.oracle.svm.core.sampler.SamplerSampleWriter;
 import com.oracle.svm.core.sampler.SamplerSampleWriterData;
 import com.oracle.svm.core.sampler.SamplerSampleWriterDataAccess;
@@ -135,6 +136,49 @@ public long getStackTraceId(int skipCount) {
         }
     }
 
+    /** Don't store to the stacktrace repo here (it may not belong in this epoch). */
+    @NeverInline("Starting a stack walk in the caller frame.")
+    @Uninterruptible(reason = "Result is only valid until epoch changes.", callerMustBe = true)
+    public boolean recordStackTraceDataToBuffer(int skipCount, SamplerBuffer rawStackTraceData) {
+        if (DeoptimizationSupport.enabled() || rawStackTraceData.isNull()) {
+            /* Stack traces are not supported if JIT compilation is used (GR-43686). */
+            return false;
+        }
+
+        /**
+         * The JFR sampler does not share the same buffers as the leak profiler. There is no risk of
+         * races between them.
+         */
+        SamplerSampleWriterData data = StackValue.get(SamplerSampleWriterData.class);
+        if (rawStackTraceData.isNull() || !SamplerSampleWriterDataAccess.initialize(data, skipCount, true, rawStackTraceData)) {
+            return false;
+        }
+
+        assert SamplerSampleWriterDataAccess.verify(data);
+        assert data.getCurrentPos().unsignedRemainder(Long.BYTES).equal(0);
+
+        /*
+         * Start a stack trace and do a stack walk. Note that the data will only be committed to the
+         * buffer if it is a new stack trace.
+         */
+        SamplerSampleWriter.begin(data);
+        Pointer sp = KnownIntrinsics.readCallerStackPointer();
+        CodePointer ip = FrameAccess.singleton().readReturnAddress(CurrentIsolate.getCurrentThread(), sp);
+        int errorCode = JfrStackWalker.walkCurrentThread(data, ip, sp, false);
+
+        /*
+         * If writing the raw data to the buffer succeeded, commit it here. Do not attempt
+         * deduplication yet since the event data this stack trace corresponds to may not be emitted
+         * until a much later epoch.
+         */
+        return switch (errorCode) {
+            case JfrStackWalker.NO_ERROR, JfrStackWalker.TRUNCATED -> SamplerSampleWriter.end(data, SamplerSampleWriter.JFR_STACK_TRACE_END);
+            case JfrStackWalker.BUFFER_SIZE_EXCEEDED -> false;
+            case JfrStackWalker.UNPARSEABLE_STACK -> throw VMError.shouldNotReachHere("Only the async sampler may encounter an unparseable stack.");
+            default -> throw VMError.shouldNotReachHere("Unexpected return value");
+        };
+    }
+
     @Uninterruptible(reason = "Result is only valid until epoch changes.", callerMustBe = true)
     private long storeDeduplicatedStackTrace(SamplerSampleWriterData data) {
         if (SamplerSampleWriter.isValid(data)) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/SubstrateJVM.java

@@ -43,6 +43,7 @@
 import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler;
 import com.oracle.svm.core.jfr.throttling.JfrEventThrottling;
 import com.oracle.svm.core.log.Log;
+import com.oracle.svm.core.sampler.SamplerBuffer;
 import com.oracle.svm.core.sampler.SamplerBufferPool;
 import com.oracle.svm.core.sampler.SamplerBuffersAccess;
 import com.oracle.svm.core.sampler.SamplerStatistics;
@@ -281,6 +282,18 @@ public long getStackTraceId(long eventTypeId, int skipCount) {
         }
     }
 
+    /*
+     * Buffers are not shared with the sampler or other JFR event types. This data will not yet be
+     * tied to a specific epoch until event emission.
+     */
+    @Uninterruptible(reason = Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public boolean recordStackTraceDataToBuffer(JfrEvent eventType, int skipCount, SamplerBuffer rawStackTraceData) {
+        if (isRecording() && isStackTraceEnabled(eventType.getId())) {
+            return stackTraceRepo.recordStackTraceDataToBuffer(skipCount, rawStackTraceData);
+        }
+        return false;
+    }
+
     /**
      * See {@link JVM#getStackTraceId}.
      */

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObject.java

@@ -32,12 +32,18 @@
 
 import jdk.graal.compiler.word.Word;
 import org.graalvm.word.UnsignedWord;
+import org.graalvm.word.WordFactory;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.collections.UninterruptibleComparable;
 import com.oracle.svm.core.collections.UninterruptibleLinkedList;
 import com.oracle.svm.core.heap.ReferenceInternals;
 import com.oracle.svm.core.jfr.JfrTicks;
+import com.oracle.svm.core.memory.NullableNativeMemory;
+import com.oracle.svm.core.nmt.NmtCategory;
+import com.oracle.svm.core.sampler.SamplerBuffer;
+import com.oracle.svm.core.sampler.SamplerBufferAccess;
+import com.oracle.svm.core.jfr.SubstrateJVM;
 
 /**
  * Holds information about a sampled object. This data may only be accessed while holding the
@@ -50,26 +56,29 @@ public final class JfrOldObject implements UninterruptibleComparable, Uninterrup
     private UnsignedWord span;
     private UnsignedWord objectSize;
     private long allocationTicks;
-    private long threadId;
-    private long stackTraceId;
     private UnsignedWord heapUsedAfterLastGC;
     private int arrayLength;
+    private SamplerBuffer rawStackTraceData;
+    private Thread thread;
+    private boolean stackTraceValid;
 
     JfrOldObject() {
         this.reference = new WeakReference<>(null);
+        this.rawStackTraceData = WordFactory.nullPointer();
     }
 
     @SuppressWarnings("hiding")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    void initialize(Object obj, UnsignedWord span, UnsignedWord allocatedSize, long threadId, long stackTraceId, UnsignedWord heapUsedAfterLastGC, int arrayLength) {
+    void initialize(Object obj, UnsignedWord span, UnsignedWord allocatedSize, Thread thread, UnsignedWord heapUsedAfterLastGC, int arrayLength) {
         ReferenceInternals.setReferent(reference, obj);
         this.span = span;
         this.objectSize = allocatedSize;
         this.allocationTicks = JfrTicks.elapsedTicks();
-        this.threadId = threadId;
-        this.stackTraceId = stackTraceId;
         this.heapUsedAfterLastGC = heapUsedAfterLastGC;
         this.arrayLength = arrayLength;
+        this.thread = thread;
+        this.stackTraceValid = false;
+        this.rawStackTraceData = WordFactory.nullPointer();
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
@@ -78,11 +87,12 @@ void reset() {
         this.span = Word.zero();
         this.objectSize = Word.zero();
         this.allocationTicks = 0L;
-        this.threadId = 0L;
-        this.stackTraceId = 0L;
         this.heapUsedAfterLastGC = Word.zero();
         this.arrayLength = 0;
         this.next = null;
+        this.thread = null;
+        this.stackTraceValid = false;
+        freeRawStackTraceData();
     }
 
     @Override
@@ -118,23 +128,53 @@ public long getAllocationTicks() {
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public long getThreadId() {
-        return threadId;
+    public UnsignedWord getHeapUsedAfterLastGC() {
+        return heapUsedAfterLastGC;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public long getStackTraceId() {
-        return stackTraceId;
+    public int getArrayLength() {
+        return arrayLength;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public UnsignedWord getHeapUsedAfterLastGC() {
-        return heapUsedAfterLastGC;
+    public Thread getThread() {
+        return thread;
     }
 
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
-    public int getArrayLength() {
-        return arrayLength;
+    public boolean getStackTraceValid() {
+        return stackTraceValid;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public void setStackTraceValid(boolean valid) {
+        stackTraceValid = valid;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    public SamplerBuffer getRawStackTraceData() {
+        if (rawStackTraceData.isNull()) {
+            // TODO add OldObjectSample event skipcount here too.
+            long bufferDataSize = Long.SIZE * SubstrateJVM.getStackTraceRepo().getStackTraceDepth();
+            rawStackTraceData = NullableNativeMemory.malloc(SamplerBufferAccess.getTotalBufferSize(WordFactory.unsigned(bufferDataSize)), NmtCategory.JFR);
+            if (rawStackTraceData.isNonNull()) {
+                rawStackTraceData.setSize(WordFactory.unsigned(bufferDataSize));
+                rawStackTraceData.setNext(Word.nullPointer());
+                SamplerBufferAccess.reinitialize(rawStackTraceData);
+            } else {
+                this.stackTraceValid = false;
+            }
+        }
+        return rawStackTraceData;
+    }
+
+    @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
+    private void freeRawStackTraceData() {
+        if (rawStackTraceData.isNonNull()) {
+            NullableNativeMemory.free(rawStackTraceData);
+            this.rawStackTraceData = WordFactory.nullPointer();
+        }
     }
 
     @Override

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObjectProfiler.java

@@ -28,6 +28,7 @@
 
 import org.graalvm.word.UnsignedWord;
 
+import com.oracle.svm.core.util.BasedOnJDKFile;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.jfr.JfrEvent;
 import com.oracle.svm.core.thread.JavaSpinLockUtils;
@@ -63,10 +64,15 @@ public void configure(int oldObjectQueueSize) {
     }
 
     public void reset() {
+        if (this.sampler != null) {
+            this.sampler.reset();
+        }
         this.sampler = new JfrOldObjectSampler(queueSize);
     }
 
     public void teardown() {
+        // Reset to free buffers.
+        this.sampler.reset();
         this.sampler = null;
     }
 
@@ -78,6 +84,7 @@ public boolean sample(Object obj, UnsignedWord allocatedSize, int arrayLength) {
         return sample0(obj, allocatedSize, arrayLength);
     }
 
+    @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+26/src/hotspot/share/jfr/leakprofiler/sampling/objectSampler.cpp#L219-L236")
     @Uninterruptible(reason = "Must not safepoint while holding the lock.")
     private boolean sample0(Object obj, UnsignedWord allocatedSize, int arrayLength) {
         assert allocatedSize.aboveThan(0);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jfr/oldobject/JfrOldObjectRepository.java

@@ -33,6 +33,7 @@
 import org.graalvm.nativeimage.StackValue;
 import org.graalvm.word.Pointer;
 
+import com.oracle.svm.core.util.BasedOnJDKFile;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue;
 import com.oracle.svm.core.jdk.UninterruptibleUtils;
@@ -102,6 +103,7 @@ public long serializeOldObject(Object obj) {
         }
     }
 
+    @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+26/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleDescription.cpp#L124-L142")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private static void writeDescription(Object obj, JfrNativeEventWriterData data) {
         if (obj instanceof ThreadGroup group) {
@@ -116,6 +118,7 @@ private static void writeDescription(Object obj, JfrNativeEventWriterData data)
         }
     }
 
+    @BasedOnJDKFile("https://github.com/openjdk/jdk/blob/jdk-24+26/src/hotspot/share/jfr/leakprofiler/checkpoint/objectSampleDescription.cpp#L50-L68")
     @Uninterruptible(reason = CALLED_FROM_UNINTERRUPTIBLE_CODE, mayBeInlined = true)
     private static void writeDescription(JfrNativeEventWriterData data, String prefix, String text) {
         if (text == null || text.isEmpty()) {