fix(profiler): mark skipped frozen virtual-thread callers as truncated

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -1922,6 +1922,17 @@ void Recording::writeThreads(Buffer *buf) {
   }
 }
 
+void Recording::writeStackTraceHeader(Buffer *buf, const CallTrace *trace,
+                                     const MethodInfo *root) {
+  buf->putVar64(trace->trace_id);
+  if (root != nullptr) {
+    bool truncated = trace->truncated ||
+                     (root->_type < FRAME_NATIVE && !root->_is_entry);
+    buf->put8(truncated);
+  }
+  buf->putVar64(trace->num_frames);
+}
+
 int Recording::writeStackTraces(Buffer *buf, Lookup *lookup) {
   // Reset all referenced flags before processing
   for (MethodMap::iterator it = _method_map.begin(); it != _method_map.end(); ++it) {
@@ -1942,18 +1953,12 @@ int Recording::writeStackTraces(Buffer *buf, Lookup *lookup) {
     buf->putVar64(traces.size());
     for (auto it = traces.begin(); it != traces.end(); ++it) {
       CallTrace *trace = *it;
-      buf->putVar64(trace->trace_id);
+      MethodInfo *root = nullptr;
       if (trace->num_frames > 0) {
-        MethodInfo *mi =
-            lookup->resolveMethod(trace->frames[trace->num_frames - 1]);
-        mi->_referenced = true;  // Mark method as referenced
-        if (mi->_type < FRAME_NATIVE) {
-          buf->put8(mi->_is_entry ? 0 : 1);
-        } else {
-          buf->put8(trace->truncated);
-        }
+        root = lookup->resolveMethod(trace->frames[trace->num_frames - 1]);
+        root->_referenced = true;
       }
-      buf->putVar64(trace->num_frames);
+      writeStackTraceHeader(buf, trace, root);
       for (int i = 0; i < trace->num_frames; i++) {
         MethodInfo *mi = lookup->resolveMethod(trace->frames[i]);
         mi->_referenced = true;  // Mark method as referenced

ddprof-lib/src/main/cpp/flightRecorder.h

@@ -37,6 +37,7 @@
 
 class VMSymbol;  // hotspot/vmStructs.h
 class ProfiledThread;
+struct CallTrace;
 
 const u64 MAX_JLONG = 0x7fffffffffffffffULL;
 const u64 MIN_JLONG = 0x8000000000000000ULL;
@@ -280,6 +281,11 @@ class Recording {
   static size_t countSerializableChildren(
       const std::vector<const Element *> &children, int depth);
 
+  // Encodes the stack-pool record header. A Java entry root can establish
+  // completeness only when the walker has not already reported missing frames.
+  static void writeStackTraceHeader(Buffer *buf, const CallTrace *trace,
+                                   const MethodInfo *root);
+
   RecordingBuffer _buf[CONCURRENCY_LEVEL];
   // we have several tables to avoid lock contention
   // we have a second dimension to allow a switch in the active table

ddprof-lib/src/main/cpp/hotspot/contBoundary.h

@@ -0,0 +1,41 @@
+/*
+ * Copyright 2026, Datadog, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#ifndef _HOTSPOT_CONTBOUNDARY_H
+#define _HOTSPOT_CONTBOUNDARY_H
+
+// Classification of a virtual-thread continuation boundary crossed by walkVM.
+// Two independent facts are derived from it:
+//   - completeness: whether the crossing proves no older Java callers remain
+//     frozen in a heap StackChunk (only Thawed does; everything else marks
+//     the sample truncated);
+//   - layout (JDK 21-26, no ContinuationEntry in vmStructs): whether the
+//     enterSpecial frame FP is the current fp (Thawed, Unverified) or the
+//     saved caller FP at *fp (Frozen).
+enum class ContBoundary : unsigned char {
+    Thawed,      // verified: bottom VT frame returns to cont_entry_return_pc / enterSpecial
+    Frozen,      // verified: bottom VT frame returns to cont_returnBarrier
+    Unverified,  // speculative: unknown CodeHeap PC on a carrier; state unknown
+};
+
+// Only a verified fully-thawed boundary proves completeness. Any other value,
+// including future enumerators, defaults to "not proven".
+constexpr bool contBoundaryProvesComplete(ContBoundary kind) {
+    return kind == ContBoundary::Thawed;
+}
+
+// JDK 21-26 entry_fp derivation: fp itself, unless verified frozen (then *fp).
+constexpr bool contBoundaryEntryFpIsCurrentFp(ContBoundary kind) {
+    return kind != ContBoundary::Frozen;
+}
+
+// Per-call-site classification used by walkVM. Call sites must pass one of
+// these, never a bare enumerator.
+inline constexpr ContBoundary CONT_BOUNDARY_AT_RETURN_BARRIER         = ContBoundary::Frozen;
+inline constexpr ContBoundary CONT_BOUNDARY_AT_ENTRY_RETURN_PC        = ContBoundary::Thawed;
+inline constexpr ContBoundary CONT_BOUNDARY_AT_ENTER_SPECIAL_NM       = ContBoundary::Thawed;
+inline constexpr ContBoundary CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM = ContBoundary::Unverified;
+
+#endif // _HOTSPOT_CONTBOUNDARY_H

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -11,6 +11,7 @@
 #include "faultInjection.h"
 #include "frames.h"
 #include "guards.h"
+#include "hotspot/contBoundary.h"
 #include "hotspot/hotspotSupport.h"
 #include "hotspot/jitCodeCache.h"
 #include "hotspot/klassTraceId.h"
@@ -417,22 +418,32 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
 
     // Advances through a continuation boundary to the carrier frame.
     // Without carrier_frames (default, cstack=vm): always stops with a "JVM Continuation"
-    // synthetic root frame — VT frames are complete, carrier internals are noise.
+    // synthetic root frame; frozen VT callers are not reconstructed.
     // With carrier_frames (cstack=vmx): attempts to walk through; failures emit BCI_ERROR
     // so the sample is truthfully marked truncated.
     // Walks cont_entry->parent() on repeated calls to handle nested continuations
     // (_parent not triggered by standard single-level VTs today, but required
     // once any runtime layers continuations on top of VTs).
     //
-    // all_frames_thawed: true when the bottom VT frame's return PC is
-    //         cont_entry_return_pc (all VT frames are thawed — CPU-bound VT),
-    //         false when it is cont_returnBarrier (frozen frames remain in the
-    //         StackChunk — VT parked and just remounted).
-    //         Needed to derive entry_fp on JDK 21-26 where ContinuationEntry
-    //         type size is absent from vmStructs and contEntry() returns nullptr.
+    // kind: how the caller classified the boundary (see hotspot/contBoundary.h).
+    //   Completeness: unless kind proves completeness (verified Thawed), older
+    //     Java callers may remain frozen in a heap StackChunk and be skipped by
+    //     this walker, so the sample is marked truncated. That covers Frozen
+    //     (return barrier) and Unverified (speculative unknown-nmethod crossing).
+    //   Layout: on JDK 21-26, where ContinuationEntry type size is absent from
+    //     vmStructs and contEntry() returns nullptr, entry_fp is fp for
+    //     Thawed/Unverified and *fp for Frozen. For Unverified this is a
+    //     best-effort thawed guess; if it is wrong, the fp/sp validation below
+    //     emits BCI_ERROR.
     //
     // Returns true to continue the walk, false to break.
-    auto walkThroughContinuation = [&](bool all_frames_thawed) -> bool {
+    auto walkThroughContinuation = [&](ContBoundary kind) -> bool {
+        // Unless the boundary is verified fully thawed, older Java callers may
+        // remain in heap StackChunks, which this walker skips even when carrier
+        // walking succeeds.
+        if (!contBoundaryProvesComplete(kind) && truncated != nullptr) {
+            *truncated = true;
+        }
         if (depth >= actual_max_depth) return false;
         if (!features.carrier_frames) {
             fillFrame(frames[depth++], BCI_NATIVE_FRAME, CONT_ROOT_FRAME);
@@ -456,13 +467,13 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         } else {
             // ContinuationEntry absent from vmStructs (JDK 21-26).
             // Derive the enterSpecial frame FP from the current fp:
-            //   all frames thawed (pc == cont_entry_return_pc): fp IS the
-            //     enterSpecial frame FP.
-            //   frozen frames remain (pc == cont_returnBarrier): the saved
-            //     caller FP at *fp leads to the enterSpecial frame on the
-            //     carrier stack.
+            //   thawed or unverified (pc == cont_entry_return_pc, or a
+            //     speculative crossing): fp is taken as the enterSpecial
+            //     frame FP.
+            //   frozen (pc == cont_returnBarrier): the saved caller FP at *fp
+            //     leads to the enterSpecial frame on the carrier stack.
             // Nested continuation tracking is unavailable without type_size().
-            entry_fp = all_frames_thawed ? fp : (uintptr_t)SafeAccess::load((void**)fp);
+            entry_fp = contBoundaryEntryFpIsCurrentFp(kind) ? fp : (uintptr_t)SafeAccess::load((void**)fp);
         }
 
         if (!StackWalkValidation::isValidFP(entry_fp)) {
@@ -519,40 +530,48 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 break;
             }
             have_prev_native_pc = false; // we are in JVM code, no previous 'native' PC
-            // Both continuation boundary PCs are JVM stubs whose findNMethod()
-            // returns NULL; detect them by exact-PC match before the nmethod
-            // dispatch below.
+            // Detect both continuation boundary PCs by exact-PC match before the
+            // nmethod dispatch below (findNMethod would otherwise classify them
+            // as ordinary stub/nmethod blobs).
             // cont_returnBarrier: bottom thawed frame returns here when frozen
             //   frames remain in the StackChunk (blocking/remounted VT).
             // cont_entry_return_pc: bottom thawed frame returns here when the
             //   continuation is fully thawed (CPU-bound VT, never yielded).
             if (!CONT_UNWIND_DISABLED && VMStructs::isContReturnBarrier(walk_pc.raw())) {
                 Counters::increment(WALKVM_CONT_BARRIER_HIT);
-                if (walkThroughContinuation(false)) continue;
+                if (walkThroughContinuation(CONT_BOUNDARY_AT_RETURN_BARRIER)) continue;
                 break;
             }
             if (!CONT_UNWIND_DISABLED && VMStructs::isContEntryReturnPc(walk_pc.raw())) {
                 Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
-                if (walkThroughContinuation(true)) continue;
+                if (walkThroughContinuation(CONT_BOUNDARY_AT_ENTRY_RETURN_PC)) continue;
                 break;
             }
             VMNMethod* nm = CodeHeap::findNMethod(walk_pc.raw());
             if (nm == NULL) {
                 // On JDK 21+ builds, the continuation entry PC may be absent
                 // from vmStructs OR resolved but pointing to the wrong address
                 // (some distributions expose the symbol at the wrong address, so
-                // the exact-PC check above never fires).  Attempt a fully-thawed
-                // continuation walk whenever we see an unknown nmethod after
-                // collecting Java frames.  walkThroughContinuation validates the
-                // fp chain and emits BCI_ERROR cleanly on mismatch, so false
-                // positives are safe.
+                // the exact-PC check above never fires).  Attempt a continuation
+                // walk whenever we see an unknown nmethod after collecting Java
+                // frames.  The crossing is UNVERIFIED: nothing here proves
+                // whether frozen frames remain in the StackChunk.  The walk uses
+                // the thawed layout guess (entry_fp = fp); the lambda's fp/sp
+                // validation emits BCI_ERROR on mismatch.  The sample is always
+                // marked truncated, because only a verified fully-thawed
+                // boundary proves completeness (a possible false positive is
+                // preferred over silently dropping frozen callers).
+                // Known limitation: for a frozen continuation reached here on
+                // JDK 21-26 the layout guess may be off by one frame.
                 if (!CONT_UNWIND_DISABLED
                         && features.carrier_frames
                         && VM::hotspot_version() >= 21
                         && depth > 0
                         && vm_thread != NULL && vm_thread->isCarryingVirtualThread()) {
                     Counters::increment(WALKVM_CONT_SPECULATIVE_HIT);
-                    if (walkThroughContinuation(true)) continue;
+                    static_assert(!contBoundaryProvesComplete(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM),
+                                  "speculative continuation crossing must mark the sample truncated");
+                    if (walkThroughContinuation(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM)) continue;
                     break;
                 }
                 if (anchor == NULL) {
@@ -649,7 +668,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 // and navigate to the carrier thread via ContinuationEntry.
                 if (!CONT_UNWIND_DISABLED && nm == VMStructs::enterSpecialNMethod()) {
                     Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
-                    if (walkThroughContinuation(true)) continue;
+                    if (walkThroughContinuation(CONT_BOUNDARY_AT_ENTER_SPECIAL_NM)) continue;
                     break;
                 }
                 // Check if deoptimization is in progress before walking compiled frames

ddprof-lib/src/main/cpp/jvmThread.h

@@ -15,6 +15,7 @@
  * JVMThread represents a native JVM thread that is JVM implementation agnostic
  */
 class JVMThread {
+    friend class JVMThreadTestAccessor;
 private:
     static jfieldID _tid;
     static ThreadLocal<JVMThread*> _jvm_thread;

ddprof-lib/src/main/cpp/threadLocal.h

@@ -187,6 +187,7 @@ class JVMThread;
  */
 template <>
 class ThreadLocal<JVMThread*> {
+    friend class JVMThreadTestAccessor;
 protected:
     pthread_key_t _key;
 

ddprof-lib/src/test/cpp/contBoundary_ut.cpp

@@ -0,0 +1,72 @@
+/*
+ * Copyright 2026, Datadog, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+// Included first: the TU fails to compile if the header is not self-contained.
+#include "hotspot/contBoundary.h"
+
+#include <gtest/gtest.h>
+
+// Compile-time checks: the predicates and per-site constants are constexpr.
+static_assert(!contBoundaryProvesComplete(ContBoundary::Frozen), "Frozen must not prove completeness");
+static_assert(!contBoundaryEntryFpIsCurrentFp(ContBoundary::Frozen), "Frozen uses *fp layout");
+static_assert(contBoundaryProvesComplete(ContBoundary::Thawed), "Thawed proves completeness");
+static_assert(contBoundaryEntryFpIsCurrentFp(ContBoundary::Thawed), "Thawed uses fp layout");
+static_assert(!contBoundaryProvesComplete(ContBoundary::Unverified), "Unverified must not prove completeness");
+static_assert(contBoundaryEntryFpIsCurrentFp(ContBoundary::Unverified), "Unverified keeps the thawed fp layout guess");
+static_assert(!contBoundaryProvesComplete(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM),
+              "speculative crossing must mark the sample truncated");
+static_assert(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM),
+              "speculative crossing keeps the thawed fp layout guess");
+static_assert(!contBoundaryProvesComplete(CONT_BOUNDARY_AT_RETURN_BARRIER),
+              "return barrier crossing must mark the sample truncated");
+static_assert(!contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_RETURN_BARRIER),
+              "return barrier crossing uses *fp layout");
+
+// R4: return barrier (frozen frames remain) marks truncated and uses *fp.
+TEST(ContBoundary, FrozenMarksTruncatedAndUsesSavedFp) {
+    EXPECT_FALSE(contBoundaryProvesComplete(ContBoundary::Frozen));
+    EXPECT_FALSE(contBoundaryEntryFpIsCurrentFp(ContBoundary::Frozen));
+}
+
+// R5/R6: verified fully-thawed boundary is complete and uses fp.
+TEST(ContBoundary, ThawedIsCompleteAndUsesCurrentFp) {
+    EXPECT_TRUE(contBoundaryProvesComplete(ContBoundary::Thawed));
+    EXPECT_TRUE(contBoundaryEntryFpIsCurrentFp(ContBoundary::Thawed));
+}
+
+// R7/R10: speculative (unverified) crossing marks truncated but keeps the
+// thawed layout guess. Pre-fix semantics treated it as "thawed => complete".
+TEST(ContBoundary, UnverifiedMarksTruncatedButKeepsThawedLayout) {
+    EXPECT_FALSE(contBoundaryProvesComplete(ContBoundary::Unverified));
+    EXPECT_TRUE(contBoundaryEntryFpIsCurrentFp(ContBoundary::Unverified));
+}
+
+// R4-R7: per-call-site constants classify each walkVM boundary.
+TEST(ContBoundary, CallSiteConstantsClassifyEachBoundary) {
+    EXPECT_EQ(CONT_BOUNDARY_AT_RETURN_BARRIER, ContBoundary::Frozen);
+    EXPECT_EQ(CONT_BOUNDARY_AT_ENTRY_RETURN_PC, ContBoundary::Thawed);
+    EXPECT_EQ(CONT_BOUNDARY_AT_ENTER_SPECIAL_NM, ContBoundary::Thawed);
+    EXPECT_EQ(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM, ContBoundary::Unverified);
+
+    EXPECT_FALSE(contBoundaryProvesComplete(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM));
+    EXPECT_TRUE(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOWN_NM));
+    EXPECT_FALSE(contBoundaryProvesComplete(CONT_BOUNDARY_AT_RETURN_BARRIER));
+    EXPECT_FALSE(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_RETURN_BARRIER));
+    EXPECT_TRUE(contBoundaryProvesComplete(CONT_BOUNDARY_AT_ENTRY_RETURN_PC));
+    EXPECT_TRUE(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_ENTRY_RETURN_PC));
+    EXPECT_TRUE(contBoundaryProvesComplete(CONT_BOUNDARY_AT_ENTER_SPECIAL_NM));
+    EXPECT_TRUE(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_ENTER_SPECIAL_NM));
+}
+
+// R9: completeness is an allow-list — only Thawed proves it; any other value,
+// including an out-of-range one, defaults to "not proven" (truncated).
+TEST(ContBoundary, OnlyThawedProvesCompleteness) {
+    const ContBoundary all[] = {ContBoundary::Thawed, ContBoundary::Frozen, ContBoundary::Unverified};
+    for (ContBoundary k : all) {
+        EXPECT_EQ(contBoundaryProvesComplete(k), k == ContBoundary::Thawed);
+    }
+    ContBoundary unknown = static_cast<ContBoundary>(0xFE);
+    EXPECT_FALSE(contBoundaryProvesComplete(unknown));
+}