fix(profiler): recover carrier frames without preserved frame pointers

ddprof-lib/src/main/cpp/flightRecorder.cpp

@@ -2314,19 +2314,29 @@ void Recording::writeCounters(Buffer *buf) {
   }
 }
 
+#ifdef DEBUG
+void Recording::writeUnwindFailureEvent(Buffer *buf, u64 ticks, const char *kind,
+                                       const char *name, u64 count) {
+  // A 255-byte name plus the other fields needs a multi-byte event size.
+  int start = buf->skip(MAX_VAR32_LENGTH);
+  buf->putVar64(T_UNWIND_FAILURE);
+  buf->putVar64(ticks);
+  buf->putUtf8(kind);
+  buf->putUtf8(name);
+  buf->putVar64(count);
+  buf->putVar32(start, buf->offset() - start);
+}
+#endif
+
 void Recording::writeUnwindFailures(Buffer *buf) {
 #ifdef DEBUG
   static UnwindFailures failures;
   UnwindStats::collectAndReset(failures);
 
   failures.forEach([&](UnwindFailureKind kind, const char *name, u64 count) {
-    int start = buf->skip(1);
-    buf->putVar64(T_UNWIND_FAILURE);
-    buf->putVar64(_start_ticks);
-    buf->putUtf8((kind & UNWIND_FAILURE_STUB) ? "stub" : "other");
-    buf->putUtf8(name);
-    buf->putVar64(count);
-    writeEventSizePrefix(buf, start);
+    writeUnwindFailureEvent(buf, _start_ticks,
+                            (kind & UNWIND_FAILURE_STUB) ? "stub" : "other",
+                            name, count);
     flushIfNeeded(buf);
   });
 #endif // DEBUG

ddprof-lib/src/main/cpp/flightRecorder.h

@@ -280,6 +280,10 @@ class Recording {
   // the encoded count can never diverge from what actually gets serialized.
   static size_t countSerializableChildren(
       const std::vector<const Element *> &children, int depth);
+#ifdef DEBUG
+  static void writeUnwindFailureEvent(Buffer *buf, u64 ticks, const char *kind,
+                                     const char *name, u64 count);
+#endif
 
   // Encodes the stack-pool record header. A Java entry root can establish
   // completeness only when the walker has not already reported missing frames.

ddprof-lib/src/main/cpp/hotspot/contBoundary.h

@@ -11,9 +11,9 @@
 //   - completeness: whether the crossing proves no older Java callers remain
 //     frozen in a heap StackChunk (only Thawed does; everything else marks
 //     the sample truncated);
-//   - layout (JDK 21-26, no ContinuationEntry in vmStructs): whether the
-//     enterSpecial frame FP is the current fp (Thawed, Unverified) or the
-//     saved caller FP at *fp (Frozen).
+//   - layout: Frozen needs the bottom frame's stack arguments for SP-based
+//     entry recovery. Without entry metadata, only Thawed/Unverified permit
+//     guessing that the current fp is the enterSpecial frame FP.
 enum class ContBoundary : unsigned char {
     Thawed,      // verified: bottom VT frame returns to cont_entry_return_pc / enterSpecial
     Frozen,      // verified: bottom VT frame returns to cont_returnBarrier
@@ -26,7 +26,7 @@ constexpr bool contBoundaryProvesComplete(ContBoundary kind) {
     return kind == ContBoundary::Thawed;
 }
 
-// JDK 21-26 entry_fp derivation: fp itself, unless verified frozen (then *fp).
+// Whether entry recovery may fall back to the current fp without entry metadata.
 constexpr bool contBoundaryEntryFpIsCurrentFp(ContBoundary kind) {
     return kind != ContBoundary::Frozen;
 }

ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp

@@ -391,6 +391,8 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
 
     // Last ContinuationEntry crossed; advanced via parent() for nested continuations.
     VMContinuationEntry* cont_entry = nullptr;
+    int continuation_stack_args = -1;
+    VMNMethod* continuation_entry_nm = VMStructs::enterSpecialNMethod();
 
     // Saved anchor data — preserved across anchor consumption so inline
     // recovery can redirect even after the anchor pointer has been set to NULL.
@@ -430,11 +432,11 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
     //     Java callers may remain frozen in a heap StackChunk and be skipped by
     //     this walker, so the sample is marked truncated. That covers Frozen
     //     (return barrier) and Unverified (speculative unknown-nmethod crossing).
-    //   Layout: on JDK 21-26, where ContinuationEntry type size is absent from
-    //     vmStructs and contEntry() returns nullptr, entry_fp is fp for
-    //     Thawed/Unverified and *fp for Frozen. For Unverified this is a
-    //     best-effort thawed guess; if it is wrong, the fp/sp validation below
-    //     emits BCI_ERROR.
+    //   Layout: without exported ContinuationEntry metadata, known boundaries
+    //     use SP, stack arguments and the generated enterSpecial frame size.
+    //     If that metadata is unavailable, Thawed/Unverified use fp as a
+    //     best-effort guess; Frozen fails rather than following an FP link
+    //     that fast thaw does not repair.
     //
     // Returns true to continue the walk, false to break.
     auto walkThroughContinuation = [&](ContBoundary kind) -> bool {
@@ -465,18 +467,21 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
             }
             entry_fp = cont_entry->entryFP();
         } else {
-            // ContinuationEntry absent from vmStructs (JDK 21-26).
-            // Derive the enterSpecial frame FP from the current fp:
-            //   thawed or unverified (pc == cont_entry_return_pc, or a
-            //     speculative crossing): fp is taken as the enterSpecial
-            //     frame FP.
-            //   frozen (pc == cont_returnBarrier): the saved caller FP at *fp
-            //     leads to the enterSpecial frame on the carrier stack.
-            // Nested continuation tracking is unavailable without type_size().
-            entry_fp = contBoundaryEntryFpIsCurrentFp(kind) ? fp : (uintptr_t)SafeAccess::load((void**)fp);
+            VMNMethod* entry_nm = continuation_entry_nm;
+            bool known_boundary = VMStructs::isContReturnBarrier(walk_pc.raw())
+                    || VMStructs::isContEntryReturnPc(walk_pc.raw())
+                    || (entry_nm != nullptr && CodeHeap::findNMethod(walk_pc.raw()) == entry_nm);
+            if (entry_nm != nullptr && known_boundary) {
+                // Fast thaw does not repair FP links. Recover the entry from SP,
+                // stack-argument padding and enterSpecial's generated frame size.
+                entry_fp = entry_nm->continuationEntryFP(sp, kind == ContBoundary::Frozen ? continuation_stack_args : 0);
+            } else {
+                entry_fp = contBoundaryEntryFpIsCurrentFp(kind) ? fp : 0;
+            }
         }
 
-        if (!StackWalkValidation::isValidFP(entry_fp)) {
+        if (!StackWalkValidation::isValidFP(entry_fp) || entry_fp < sp ||
+                entry_fp >= bottom || bottom - entry_fp <= (FRAME_PC_SLOT + 1) * sizeof(void*)) {
             fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_fp");
             return false;
         }
@@ -487,7 +492,14 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
         uintptr_t carrier_fp = *carrier_fp_addr;
         const void* carrier_pc = ((const void**)carrier_fp_addr)[FRAME_PC_SLOT];
         uintptr_t carrier_sp = entry_fp + (FRAME_PC_SLOT + 1) * sizeof(void*);
-        if (!StackWalkValidation::isValidFP(carrier_fp) ||
+        // Compiled carrier code can use FP as a general-purpose register.
+        // Its sender is recovered from nmethod frame metadata instead.
+        bool valid_carrier_fp = StackWalkValidation::isValidFP(carrier_fp);
+        if (!valid_carrier_fp) {
+            VMNMethod* carrier_nm = CodeHeap::findNMethod(carrier_pc);
+            valid_carrier_fp = carrier_nm != nullptr && carrier_nm->isNMethod();
+        }
+        if (!valid_carrier_fp ||
             StackWalkValidation::inDeadZone(carrier_pc) ||
             !StackWalkValidation::isValidSP(carrier_sp, sp, bottom)) {
             fillFrame(frames[depth++], BCI_ERROR, "break_cont_carrier_sp");
@@ -561,8 +573,8 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 // marked truncated, because only a verified fully-thawed
                 // boundary proves completeness (a possible false positive is
                 // preferred over silently dropping frozen callers).
-                // Known limitation: for a frozen continuation reached here on
-                // JDK 21-26 the layout guess may be off by one frame.
+                // A frozen continuation reached here can fail carrier recovery
+                // because the unverified crossing uses the thawed layout guess.
                 if (!CONT_UNWIND_DISABLED
                         && features.carrier_frames
                         && VM::hotspot_version() >= 21
@@ -617,6 +629,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
             }
 
             if (nm->isInterpreter()) {
+                continuation_stack_args = 0;
                 if (vm_thread != NULL && vm_thread->inDeopt()) {
                     fillFrame(frames[depth++], BCI_ERROR, "break_deopt");
                     break;
@@ -666,7 +679,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 // continuation entry stub on JDK 27+. It has no JavaCallWrapper, so
                 // isEntryFrame() will not fire for it. Detect it by identity
                 // and navigate to the carrier thread via ContinuationEntry.
-                if (!CONT_UNWIND_DISABLED && nm == VMStructs::enterSpecialNMethod()) {
+                if (!CONT_UNWIND_DISABLED && nm == continuation_entry_nm) {
                     Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
                     if (walkThroughContinuation(CONT_BOUNDARY_AT_ENTER_SPECIAL_NM)) continue;
                     break;
@@ -685,6 +698,14 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                 jmethodID method_id = method->id();
                 HotspotSupport::fillJavaFrame(frames[depth++], type, 0, method_id, method);
 
+                // Prologue/epilogue unwinds can reach a return barrier without the frame-size step.
+                if (features.carrier_frames && VMContinuationEntry::type_size() == 0) {
+                    VMConstMethod* const_method = method->constMethod_or_null();
+                    continuation_stack_args = const_method != nullptr ? const_method->stackArgWords() : -1;
+                    // nmethod sets its stack-argument count to zero for OSR entries.
+                    if (continuation_stack_args > 0 && nm->isOsr()) continuation_stack_args = 0;
+                }
+
                 if (nm->isFrameCompleteAt(walk_pc.raw())) {
                     const void* epilogue_pc = walk_pc.raw();
                     if (depth == 1 && frame.unwindEpilogue(nm, (uintptr_t&)epilogue_pc, sp, fp)) {
@@ -922,6 +943,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                             const char* bcp = ((const char**)recovery_fp)[bcp_offset];
                             int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start;
                             HotspotSupport::fillJavaFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id, method);
+                            continuation_stack_args = 0;
                             sp = ((uintptr_t*)recovery_fp)[InterpreterFrame::sender_sp_offset];
                             walk_pc.setReturnAddress(stripPointer(((void**)recovery_fp)[FRAME_PC_SLOT]));
                             fp = *(uintptr_t*)recovery_fp;
@@ -1021,6 +1043,7 @@ __attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontex
                     const char* bcp = ((const char**)anchor_fp)[bcp_offset];
                     int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start;
                     HotspotSupport::fillJavaFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id, method);
+                    continuation_stack_args = 0;
                     sp = ((uintptr_t*)anchor_fp)[InterpreterFrame::sender_sp_offset];
                     walk_pc.setReturnAddress(stripPointer(((void**)anchor_fp)[FRAME_PC_SLOT]));
                     fp = *(uintptr_t*)anchor_fp;

ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp

@@ -420,7 +420,10 @@ void VMStructs::resolveOffsets() {
     }
     if (_cont_entry_return_pc == nullptr && VM::hotspot_version() >= 21) {
         const void** sym = (const void**)_libjvm->findSymbol("_ZN17ContinuationEntry10_return_pcE");
-        if (sym != nullptr) _cont_entry_return_pc = *sym;
+        if (sym != nullptr) {
+            _cont_entry_return_pc_addr = sym;
+            _cont_entry_return_pc = *sym;
+        }
     }
 
     // Since JDK 23, _metadata_offset is relative to _data_offset. See metadata()
@@ -472,11 +475,6 @@ void VMStructs::resolveOffsets() {
         _interpreter_nm = CodeHeap::findNMethod(_interpreter_start);
     }
     if (_enter_special_nm == NULL && _cont_entry_return_pc != NULL) {
-        // On JDK 27+, enterSpecial is a proper nmethod; findNMethod succeeds.
-        // On JDK 21-26, it is a RuntimeBlob; findNMethod returns NULL and
-        // _enter_special_nm stays NULL.  The cont_entry_return_pc boundary is
-        // then detected via isContEntryReturnPc() in the walk loop rather than
-        // nmethod identity.
         _enter_special_nm = CodeHeap::findNMethod(_cont_entry_return_pc);
     }
 }

ddprof-lib/src/main/cpp/hotspot/vmStructs.h

@@ -202,6 +202,7 @@ typedef void* address;
         field_with_version(_nmethod_entry_address, offset, 8, 22, MATCH_SYMBOLS("_verified_entry_point"))           \
         field(_nmethod_state_offset, offset, MATCH_SYMBOLS("_state"))                                               \
         field(_nmethod_level_offset, offset, MATCH_SYMBOLS("_comp_level"))                                          \
+        field_with_version(_nmethod_entry_bci_offset, offset, 21, MAX_VERSION, MATCH_SYMBOLS("_entry_bci"))           \
         field_with_version(_nmethod_metadata_offset, offset, MIN_VERSION, 24, MATCH_SYMBOLS("_metadata_offset"))    \
         field_with_version(_nmethod_immutable_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_immutable_data"))    \
         field(_scopes_pcs_offset, offset, MATCH_SYMBOLS("_scopes_pcs_offset"))                                      \
@@ -219,6 +220,7 @@ typedef void* address;
         field(_constmethod_sig_index_offset, offset, MATCH_SYMBOLS("_signature_index"))                             \
         field(_constmethod_flags_offset, offset, MATCH_SYMBOLS("_flags", "_flags._flags"))                          \
         field(_constmethod_code_size_offset, offset, MATCH_SYMBOLS("_code_size"))                                   \
+        field_with_version(_constmethod_stack_arg_slots_offset, offset, 21, MAX_VERSION, MATCH_SYMBOLS("_num_stack_arg_slots")) \
     type_end()                                                                                                      \
     type_begin(VMConstantPool, MATCH_SYMBOLS("ConstantPool"))                                                       \
         field(_pool_holder_offset, offset, MATCH_SYMBOLS("_pool_holder"))                                           \
@@ -525,14 +527,18 @@ class VMStructs {
     // True when the bottom VT frame's return PC is cont_entry_return_pc, meaning all
     // VT frames are thawed (CPU-bound VT that never yielded).
     // Available on JDK 21+ via vmStructs or symbol fallback.
-    static bool isContEntryReturnPc(const void* pc) {
-        return _cont_entry_return_pc != nullptr && pc == _cont_entry_return_pc;
+    static const void* contEntryReturnPc() {
+        return _cont_entry_return_pc_addr != nullptr
+                ? __atomic_load_n((const void**)_cont_entry_return_pc_addr, __ATOMIC_ACQUIRE) : _cont_entry_return_pc;
     }
 
-    static VMNMethod* enterSpecialNMethod() {
-        return _enter_special_nm;
+    static bool isContEntryReturnPc(const void* pc) {
+        const void* return_pc = contEntryReturnPc();
+        return return_pc != nullptr && pc == return_pc;
     }
 
+    static VMNMethod* enterSpecialNMethod();
+
     // Datadog-specific extensions
     static bool isSafeToWalk(uintptr_t pc);
     static void JNICALL NativeMethodBind(jvmtiEnv *jvmti, JNIEnv *jni,
@@ -983,6 +989,12 @@ DECLARE(VMConstantPool)
 DECLARE_END
 
 DECLARE(VMConstMethod)
+    // ConstMethod stores 4-byte VMReg slots; compiled frames round to whole machine words.
+    int stackArgWords() const {
+        if (_constmethod_stack_arg_slots_offset < 0) return -1;
+        unsigned short slots = *(const unsigned short*)at(_constmethod_stack_arg_slots_offset);
+        return (slots + 1) / 2;
+    }
     inline VMConstantPool* constants_or_null() const;
     inline VMSymbol* name() const;
     inline VMSymbol* signature() const;
@@ -1035,6 +1047,26 @@ static inline bool startsWith(const char* s, const char (&pattern)[N]) {
 }
 
 DECLARE(VMNMethod)
+    bool isOsr() const {
+        assert(_nmethod_entry_bci_offset >= 0);
+        return *(const int*)at(_nmethod_entry_bci_offset) != -1;
+    }
+
+    // ContinuationEntry::bottom_sender_sp() subtracts stack arguments and aligns down
+    // to 16 bytes. enterSpecial's frame size includes its saved FP and return PC.
+    uintptr_t continuationEntryFP(uintptr_t sp, int stack_args) {
+        int frame_size = frameSize();
+        // A reference occupies one of the JVM's 255 parameter slots but a full
+        // machine word on the stack; allow that limit plus alignment padding.
+        if (frame_size < 4 || frame_size > 256 || (frame_size & 1) != 0 ||
+                stack_args < 0 || stack_args > 256 || (sp & 15) != 0) {
+            return 0;
+        }
+        uintptr_t words = ((stack_args + 1) & ~1) + frame_size - 2;
+        if (sp > UINTPTR_MAX - words * sizeof(uintptr_t)) return 0;
+        return sp + words * sizeof(uintptr_t);
+    }
+
     int size() {
         assert(_blob_size_offset >= 0);
         return *(int*) at(_blob_size_offset);

ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h

@@ -21,6 +21,13 @@ inline bool crashProtectionActive() {
     return false;
 }
 
+inline VMNMethod* VMStructs::enterSpecialNMethod() {
+    if (_enter_special_nm != nullptr) return _enter_special_nm;
+    // The native wrapper can be generated after profiler initialization.
+    const void* pc = contEntryReturnPc();
+    return pc != nullptr ? CodeHeap::findNMethod(pc) : nullptr;
+}
+
 template <typename T>
 inline T* cast_to(const void* ptr) {
     assert(VM::isHotspot()); // This should only be used in HotSpot-specific code

ddprof-lib/src/test/cpp/contBoundary_ut.cpp

@@ -10,7 +10,7 @@
 
 // Compile-time checks: the predicates and per-site constants are constexpr.
 static_assert(!contBoundaryProvesComplete(ContBoundary::Frozen), "Frozen must not prove completeness");
-static_assert(!contBoundaryEntryFpIsCurrentFp(ContBoundary::Frozen), "Frozen uses *fp layout");
+static_assert(!contBoundaryEntryFpIsCurrentFp(ContBoundary::Frozen), "Frozen rejects the FP fallback");
 static_assert(contBoundaryProvesComplete(ContBoundary::Thawed), "Thawed proves completeness");
 static_assert(contBoundaryEntryFpIsCurrentFp(ContBoundary::Thawed), "Thawed uses fp layout");
 static_assert(!contBoundaryProvesComplete(ContBoundary::Unverified), "Unverified must not prove completeness");
@@ -22,10 +22,10 @@ static_assert(contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_SPECULATIVE_UNKNOW
 static_assert(!contBoundaryProvesComplete(CONT_BOUNDARY_AT_RETURN_BARRIER),
               "return barrier crossing must mark the sample truncated");
 static_assert(!contBoundaryEntryFpIsCurrentFp(CONT_BOUNDARY_AT_RETURN_BARRIER),
-              "return barrier crossing uses *fp layout");
+              "return barrier crossing rejects the FP fallback");
 
-// R4: return barrier (frozen frames remain) marks truncated and uses *fp.
-TEST(ContBoundary, FrozenMarksTruncatedAndUsesSavedFp) {
+// R4: a return barrier marks truncation and requires entry metadata.
+TEST(ContBoundary, FrozenMarksTruncatedAndRejectsFpFallback) {
     EXPECT_FALSE(contBoundaryProvesComplete(ContBoundary::Frozen));
     EXPECT_FALSE(contBoundaryEntryFpIsCurrentFp(ContBoundary::Frozen));
 }