fix(walkvm): unwind through virtual thread continuation boundaries

Fixes walkVM to correctly traverse JVM virtual thread (Project Loom)
continuation boundaries, exposing carrier thread frames in wall-clock
profiles.

Two unwind paths are implemented:
- Path A (enterSpecial): CPU-bound VTs that never yield — all frames
  are thawed; the profiler traverses the enterSpecial nmethod by
  identity to reach carrier frames via ContinuationEntry.
- Path B (cont_returnBarrier): blocking VTs that park/unpark — when
  remounted with frozen frames in the StackChunk, cont_returnBarrier
  is the return PC of the bottommost thawed frame. Checked before
  CodeHeap::findNMethod() since it is a JVM stub, not an nmethod.

By default a synthetic "JVM Continuation" root frame (BCI_NATIVE_FRAME)
is inserted at the boundary so the sample is not marked truncated.
With wextend=vt_carrier the profiler walks through to carrier frames;
failures emit BCI_ERROR (truthful truncation). The wextend argument is
string-parsed and extensible for future flags.

Additional changes:
- Add carrier_frames bit to StackWalkFeatures (uses one padding bit)
- Use FRAME_PC_SLOT for architecture-portable carrier frame extraction
- Split VMContinuationEntry into DECLARE_V21_TYPES_DO to prevent
  assert(type_size() > 0) on JDK <21 debug builds; expand at all
  four declare/init/read/verify sites
- Three new counters: WALKVM_CONT_BARRIER_HIT, WALKVM_ENTER_SPECIAL_HIT,
  WALKVM_CONT_ENTRY_NULL
- isValidFP / isValidSP helpers with unit tests in stackWalker_ut.cpp
- DDPROF_DISABLE_CONT_UNWIND env var for negative testing (DEBUG only)
- Integration tests: VirtualThreadWallClockTest covers both paths on
  JDK 21+ with wextend=vt_carrier

Resolves SCP-1110

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: jbachorik

ddprof-lib/src/main/cpp/arguments.cpp

@@ -300,6 +300,19 @@ Error Arguments::parse(const char *args) {
         }
       }
 
+      CASE("wextend")
+      if (value != NULL) {
+        const char* p = value;
+        while (*p != '\0') {
+          const char* comma = strchr(p, ',');
+          size_t len = comma != NULL ? (size_t)(comma - p) : strlen(p);
+          if (len == 10 && strncmp(p, "vt_carrier", 10) == 0) {
+            _features.carrier_frames = 1;
+          }
+          p += len + (comma != NULL ? 1 : 0);
+        }
+      }
+
       CASE("attributes")
       if (value != NULL) {
         std::string input(value);

ddprof-lib/src/main/cpp/arguments.h

@@ -122,7 +122,8 @@ struct StackWalkFeatures {
     unsigned short vtable_target : 1;  // show receiver classes of vtable/itable stubs
     unsigned short comp_task     : 1;  // display current compilation task for JIT threads
     unsigned short pc_addr       : 1;  // record exact PC address for each sample
-    unsigned short _padding      : 3;  // pad structure to 16 bits
+    unsigned short carrier_frames: 1;  // walk through VT continuation boundary to carrier frames (wextend=vt_carrier)
+    unsigned short _padding      : 2;  // pad structure to 16 bits
 };
 
 struct Multiplier {

ddprof-lib/src/main/cpp/counters.h

@@ -92,7 +92,10 @@
   X(WALKVM_STUB_FRAMESIZE_FALLBACK, "walkvm_stub_framesize_fallback")        \
   X(WALKVM_FP_CHAIN_ATTEMPT, "walkvm_fp_chain_attempt")                      \
   X(WALKVM_FP_CHAIN_REACHED_CODEHEAP, "walkvm_fp_chain_reached_codeheap")    \
-  X(WALKVM_ANCHOR_NOT_IN_JAVA, "walkvm_anchor_not_in_java")                    \
+  X(WALKVM_ANCHOR_NOT_IN_JAVA,  "walkvm_anchor_not_in_java")                   \
+  X(WALKVM_CONT_BARRIER_HIT,    "walkvm_cont_barrier_hit")                     \
+  X(WALKVM_ENTER_SPECIAL_HIT,   "walkvm_enter_special_hit")                    \
+  X(WALKVM_CONT_ENTRY_NULL,     "walkvm_cont_entry_null")                      \
   X(NATIVE_LIBS_DROPPED, "native_libs_dropped")                                \
   X(SIGACTION_PATCHED_LIBS, "sigaction_patched_libs")                          \
   X(SIGACTION_INTERCEPTED, "sigaction_intercepted")

ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp

@@ -35,6 +35,9 @@ int VMStructs::_narrow_klass_shift = -1;
 int VMStructs::_interpreter_frame_bcp_offset = 0;
 unsigned char VMStructs::_unsigned5_base = 0;
 const void* VMStructs::_call_stub_return = nullptr;
+const void* VMStructs::_cont_return_barrier   = nullptr;
+const void* VMStructs::_cont_entry_return_pc  = nullptr;
+VMNMethod*  VMStructs::_enter_special_nm       = nullptr;
 const void* VMStructs::_interpreter_start = nullptr;
 VMNMethod* VMStructs::_interpreter_nm = nullptr;
 const void* VMStructs::_interpreted_frame_valid_start = nullptr;
@@ -44,6 +47,7 @@ const void* VMStructs::_interpreted_frame_valid_end = nullptr;
 // Initialize type size to 0
 #define INIT_TYPE_SIZE(name, names) uint64_t VMStructs::TYPE_SIZE_NAME(name) = 0;
 DECLARE_TYPES_DO(INIT_TYPE_SIZE)
+DECLARE_V21_TYPES_DO(INIT_TYPE_SIZE)
 #undef INIT_TYPE_SIZE
 
 #define offset_value -1
@@ -205,6 +209,7 @@ void VMStructs::init_type_sizes() {
         }
 
         DECLARE_TYPES_DO(READ_TYPE_SIZE)
+        DECLARE_V21_TYPES_DO(READ_TYPE_SIZE)
 
 #undef READ_TYPE_SIZE   
 
@@ -273,6 +278,9 @@ void VMStructs::verify_offsets() {
 // Verify type sizes
 #define VERIFY_TYPE_SIZE(name, names) assert(TYPE_SIZE_NAME(name) > 0);
     DECLARE_TYPES_DO(VERIFY_TYPE_SIZE);
+    if (hotspot_version >= 21) {
+        DECLARE_V21_TYPES_DO(VERIFY_TYPE_SIZE);
+    }
 #undef VERIFY_TYPE_SIZE
 
 
@@ -391,6 +399,12 @@ void VMStructs::resolveOffsets() {
     if (_call_stub_return_addr != NULL) {
         _call_stub_return = *(const void**)_call_stub_return_addr;
     }
+    if (_cont_return_barrier_addr != NULL) {
+        _cont_return_barrier = *(const void**)_cont_return_barrier_addr;
+    }
+    if (_cont_entry_return_pc_addr != NULL) {
+        _cont_entry_return_pc = *(const void**)_cont_entry_return_pc_addr;
+    }
 
     // Since JDK 23, _metadata_offset is relative to _data_offset. See metadata()
     if (_nmethod_immutable_offset < 0) {
@@ -440,6 +454,14 @@ void VMStructs::resolveOffsets() {
     if (_interpreter_nm == NULL && _interpreter_start != NULL) {
         _interpreter_nm = CodeHeap::findNMethod(_interpreter_start);
     }
+    if (_enter_special_nm == NULL && _cont_entry_return_pc != NULL) {
+        // enterSpecial is a generated nmethod. If findNMethod returns NULL the
+        // CodeHeap may not yet contain it at VM-ready time; Path A (enterSpecial
+        // detection) will silently not trigger for those samples, which is safe —
+        // the stack will simply be truncated at the continuation boundary rather
+        // than showing carrier frames.
+        _enter_special_nm = CodeHeap::findNMethod(_cont_entry_return_pc);
+    }
 }
 
 void VMStructs::initJvmFunctions() {

ddprof-lib/src/main/cpp/hotspot/vmStructs.h

@@ -112,16 +112,20 @@ inline T* cast_to(const void* ptr) {
  */
 
 #define DECLARE_TYPES_DO(f) \
-    f(VMClassLoaderData,    MATCH_SYMBOLS("ClassLoaderData"))   \
-    f(VMConstantPool,       MATCH_SYMBOLS("ConstantPool"))      \
-    f(VMConstMethod,        MATCH_SYMBOLS("ConstMethod"))       \
-    f(VMFlag,               MATCH_SYMBOLS("JVMFlag", "Flag"))   \
-    f(VMJavaFrameAnchor,    MATCH_SYMBOLS("JavaFrameAnchor"))   \
-    f(VMKlass,              MATCH_SYMBOLS("Klass"))             \
-    f(VMMethod,             MATCH_SYMBOLS("Method"))            \
-    f(VMNMethod,            MATCH_SYMBOLS("nmethod"))           \
-    f(VMSymbol,             MATCH_SYMBOLS("Symbol"))            \
-    f(VMThread,             MATCH_SYMBOLS("Thread"))
+    f(VMClassLoaderData,      MATCH_SYMBOLS("ClassLoaderData"))   \
+    f(VMConstantPool,         MATCH_SYMBOLS("ConstantPool"))      \
+    f(VMConstMethod,          MATCH_SYMBOLS("ConstMethod"))       \
+    f(VMFlag,                 MATCH_SYMBOLS("JVMFlag", "Flag"))   \
+    f(VMJavaFrameAnchor,      MATCH_SYMBOLS("JavaFrameAnchor"))   \
+    f(VMKlass,                MATCH_SYMBOLS("Klass"))             \
+    f(VMMethod,               MATCH_SYMBOLS("Method"))            \
+    f(VMNMethod,              MATCH_SYMBOLS("nmethod"))           \
+    f(VMSymbol,               MATCH_SYMBOLS("Symbol"))            \
+    f(VMThread,               MATCH_SYMBOLS("Thread"))
+
+// Types only present in JDK 21+ (Project Loom); size is 0 on older JDKs
+#define DECLARE_V21_TYPES_DO(f) \
+    f(VMContinuationEntry,    MATCH_SYMBOLS("ContinuationEntry"))
 
 /**
  * Following macros define field offsets, addresses or values of JVM classes that are exported by
@@ -190,6 +194,7 @@ typedef void* address;
         field(_thread_anchor_offset, offset, MATCH_SYMBOLS("_anchor"))                                              \
         field(_thread_state_offset, offset, MATCH_SYMBOLS("_thread_state"))                                         \
         field(_thread_vframe_offset, offset, MATCH_SYMBOLS("_vframe_array_head"))                                   \
+        field_with_version(_cont_entry_offset, offset, 21, MAX_VERSION, MATCH_SYMBOLS("_cont_entry"))               \
     type_end()                                                                                                      \
     type_begin(VMOSThread, MATCH_SYMBOLS("OSThread"))                                                               \
         field(_osthread_id_offset, offset, MATCH_SYMBOLS("_thread_id"))                                             \
@@ -246,7 +251,12 @@ typedef void* address;
         field(_vs_high_offset, offset, MATCH_SYMBOLS("_high"))                                                      \
     type_end()                                                                                                      \
     type_begin(VMStubRoutine, MATCH_SYMBOLS("StubRoutines"))                                                        \
-        field(_call_stub_return_addr, address, MATCH_SYMBOLS("_call_stub_return_address"))                          \
+        field(_call_stub_return_addr, address, MATCH_SYMBOLS("_call_stub_return_address"))                         \
+        field_with_version(_cont_return_barrier_addr, address, 21, MAX_VERSION, MATCH_SYMBOLS("_cont_returnBarrier")) \
+    type_end()                                                                                                      \
+    type_begin(VMContinuationEntry, MATCH_SYMBOLS("ContinuationEntry"))                                            \
+        field_with_version(_cont_entry_return_pc_addr, address, 21, MAX_VERSION, MATCH_SYMBOLS("_return_pc"))       \
+        field_with_version(_cont_entry_parent_offset,  offset,  21, MAX_VERSION, MATCH_SYMBOLS("_parent"))          \
     type_end()                                                                                                      \
     type_begin(VMGrowableArray, MATCH_SYMBOLS("GrowableArrayBase", "GenericGrowableArray"))                         \
         field(_array_len_offset, offset, MATCH_SYMBOLS("_len"))                                                     \
@@ -309,6 +319,9 @@ class VMStructs {
     static int _interpreter_frame_bcp_offset;
     static unsigned char _unsigned5_base;
     static const void* _call_stub_return;
+    static const void* _cont_return_barrier;
+    static const void* _cont_entry_return_pc;
+    static VMNMethod*  _enter_special_nm;
     static const void* _interpreter_start;
     static VMNMethod* _interpreter_nm;
     static const void* _interpreted_frame_valid_start;
@@ -320,6 +333,7 @@ class VMStructs {
     static uint64_t TYPE_SIZE_NAME(name);
 
     DECLARE_TYPES_DO(DECLARE_TYPE_SIZE_VAR)
+    DECLARE_V21_TYPES_DO(DECLARE_TYPE_SIZE_VAR)
 #undef DECLARE_TYPE_SIZE_VAR
 
 // Declare vmStructs' field offsets and addresses
@@ -444,6 +458,14 @@ class VMStructs {
         return pc >= _interpreted_frame_valid_start && pc < _interpreted_frame_valid_end;
     }
 
+    static bool isContReturnBarrier(const void* pc) {
+        return _cont_return_barrier != nullptr && pc == _cont_return_barrier;
+    }
+
+    static VMNMethod* enterSpecialNMethod() {
+        return _enter_special_nm;
+    }
+
     // Datadog-specific extensions
     static bool isSafeToWalk(uintptr_t pc);
     static void JNICALL NativeMethodBind(jvmtiEnv *jvmti, JNIEnv *jni,
@@ -671,6 +693,30 @@ DECLARE(VMJavaFrameAnchor)
     }
 DECLARE_END
 
+DECLARE(VMContinuationEntry)
+  public:
+    // Address of the enterSpecial frame's {saved_fp, return_addr} pair.
+    // Layout above this address: [saved_fp][return_addr_to_carrier][carrier_sp...]
+    // The ContinuationEntry struct is embedded on the carrier stack immediately
+    // below enterSpecial's saved-fp slot; its size() equals the JVM's
+    // ContinuationEntry::size() static method, confirmed at:
+    //   https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.hpp
+    //   https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.cpp
+    uintptr_t entryFP() const {
+        assert(type_size() > 0); // must not be called before ContinuationEntry is resolved
+        return (uintptr_t)this + type_size();
+    }
+
+    // Returns the enclosing ContinuationEntry when continuations are nested
+    // (e.g. a Continuation.run() call inside a virtual thread).  Returns
+    // nullptr when there is no enclosing entry or the field is unavailable.
+    VMContinuationEntry* parent() const {
+        if (_cont_entry_parent_offset < 0) return nullptr;
+        void* ptr = SafeAccess::loadPtr((void**) at(_cont_entry_parent_offset), nullptr);
+        return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr;
+    }
+DECLARE_END
+
 // Copied from JDK's globalDefinitions.hpp 'JavaThreadState' enum
 enum JVMJavaThreadState {
     _thread_uninitialized     =  0, // should never happen (missing initialization)
@@ -790,6 +836,12 @@ DECLARE(VMThread)
         return VMJavaFrameAnchor::cast(at(_thread_anchor_offset));
     }
 
+    VMContinuationEntry* contEntry() {
+        if (_cont_entry_offset < 0) return nullptr;
+        void* ptr = SafeAccess::loadPtr((void**) at(_cont_entry_offset), nullptr);
+        return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr;
+    }
+
     inline VMMethod* compiledMethod();
 private:
     static inline int nativeThreadId(JNIEnv* jni, jthread thread);

ddprof-lib/src/main/cpp/stackWalker.cpp

@@ -4,6 +4,7 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+#include <cstdlib>
 #include <setjmp.h>
 #include "stackWalker.h"
 #include "dwarf.h"
@@ -19,6 +20,16 @@
 const uintptr_t MAX_WALK_SIZE = 0x100000;
 const intptr_t MAX_FRAME_SIZE_WORDS = StackWalkValidation::MAX_FRAME_SIZE / sizeof(void*);  // 0x8000 = 32768 words
 
+#ifdef NDEBUG
+static const bool CONT_UNWIND_DISABLED = false;
+#else
+// DEBUG-only: when set, both continuation-unwind paths (Path A: enterSpecial,
+// Path B: cont_returnBarrier) are skipped, reproducing pre-fix behaviour.
+// Used by negative integration tests to verify that carrier frames are not
+// visible and walk-error sentinels do appear without the fix.
+static const bool CONT_UNWIND_DISABLED = (std::getenv("DDPROF_DISABLE_CONT_UNWIND") != nullptr);
+#endif
+
 static ucontext_t empty_ucontext{};
 
 // Use validation helpers from header (shared with tests)
@@ -323,6 +334,9 @@ __attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext,
 
     const void* prev_native_pc = NULL;
 
+    // Last ContinuationEntry crossed; advanced via parent() for nested continuations.
+    VMContinuationEntry* cont_entry = nullptr;
+
     // Saved anchor data — preserved across anchor consumption so inline
     // recovery can redirect even after the anchor pointer has been set to NULL.
     // Recovery is one-shot: once attempted, we do not retry to avoid
@@ -339,6 +353,51 @@ __attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext,
         anchor = vm_thread->anchor();
     }
 
+    static const char* CONT_ROOT_FRAME = "JVM Continuation";
+
+    // Advances through a continuation boundary to the carrier frame.
+    // Without wextend=vt_carrier (default): always stops with a "JVM Continuation"
+    // synthetic root frame — VT frames are complete, carrier internals are noise.
+    // With wextend=carrier: attempts to walk through; failures emit BCI_ERROR
+    // so the sample is truthfully marked truncated.
+    // Walks cont_entry->parent() on repeated calls to handle nested continuations
+    // (_parent not triggered by standard single-level VTs today, but required
+    // once any runtime layers continuations on top of VTs).
+    // Returns true to continue the walk, false to break.
+    auto walkThroughContinuation = [&]() -> bool {
+        if (!features.carrier_frames) {
+            fillFrame(frames[depth++], BCI_NATIVE_FRAME, CONT_ROOT_FRAME);
+            return false;
+        }
+        cont_entry = (cont_entry != nullptr) ? cont_entry->parent() : vm_thread->contEntry();
+        if (cont_entry == nullptr) {
+            Counters::increment(WALKVM_CONT_ENTRY_NULL);
+            fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_null");
+            return false;
+        }
+        uintptr_t entry_fp = cont_entry->entryFP();
+        if (!StackWalkValidation::isValidFP(entry_fp)) {
+            fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_fp");
+            return false;
+        }
+        // entry_fp has been range-checked by isValidFP above; any remaining
+        // SIGSEGV from a stale/concurrently-freed pointer is caught by the
+        // setjmp crash protection in walkVM (checkFault -> longjmp).
+        uintptr_t carrier_fp = *(uintptr_t*)entry_fp;
+        const void* carrier_pc = ((const void**)entry_fp)[FRAME_PC_SLOT];
+        uintptr_t carrier_sp = entry_fp + (FRAME_PC_SLOT + 1) * sizeof(void*);
+        if (!StackWalkValidation::isValidFP(carrier_fp) ||
+            StackWalkValidation::inDeadZone(carrier_pc) ||
+            !StackWalkValidation::isValidSP(carrier_sp, sp, bottom)) {
+            fillFrame(frames[depth++], BCI_ERROR, "break_cont_carrier_sp");
+            return false;
+        }
+        sp = carrier_sp;
+        fp = carrier_fp;
+        pc = carrier_pc;
+        return true;
+    };
+
     unwind_loop:
 
     // Walk until the bottom of the stack or until the first Java frame
@@ -369,6 +428,15 @@ __attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext,
                 break;
             }
             prev_native_pc = NULL; // we are in JVM code, no previous 'native' PC
+
+            // cont_returnBarrier is a JVM stub — CodeHeap::findNMethod() returns NULL for it,
+            // so it must be handled before the nmethod dispatch below.
+            if (!CONT_UNWIND_DISABLED && VMStructs::isContReturnBarrier(pc)) {
+                Counters::increment(WALKVM_CONT_BARRIER_HIT);
+                if (walkThroughContinuation()) continue;
+                break;
+            }
+
             VMNMethod* nm = CodeHeap::findNMethod(pc);
             if (nm == NULL) {
                 if (anchor == NULL) {
@@ -444,6 +512,15 @@ __attribute__((no_sanitize("address"))) int StackWalker::walkVM(void* ucontext,
                 fillFrame(frames[depth++], BCI_ERROR, "break_interpreted");
                 break;
             } else if (nm->isNMethod()) {
+                // enterSpecial is a generated native nmethod that acts as the
+                // continuation entry stub. It has no JavaCallWrapper, so
+                // isEntryFrame() will not fire for it. Detect it by identity
+                // and navigate to the carrier thread via ContinuationEntry.
+                if (!CONT_UNWIND_DISABLED && nm == VMStructs::enterSpecialNMethod()) {
+                    Counters::increment(WALKVM_ENTER_SPECIAL_HIT);
+                    if (walkThroughContinuation()) continue;
+                    break;
+                }
                 // Check if deoptimization is in progress before walking compiled frames
                 if (vm_thread != NULL && vm_thread->inDeopt()) {
                     fillFrame(frames[depth++], BCI_ERROR, "break_deopt_compiled");

ddprof-lib/src/main/cpp/stackWalker.h

@@ -52,6 +52,16 @@ namespace StackWalkValidation {
         return (uintptr_t)hi - (uintptr_t)lo < SAME_STACK_DISTANCE;
     }
 
+    // Check if a frame pointer is plausibly valid (not in dead zone, properly aligned)
+    static inline bool isValidFP(uintptr_t fp) {
+        return !inDeadZone((const void*)fp) && aligned(fp);
+    }
+
+    // Check if a stack pointer is within [lo, hi) and properly aligned
+    static inline bool isValidSP(uintptr_t sp, uintptr_t lo, uintptr_t hi) {
+        return sp > lo && sp < hi && aligned(sp);
+    }
+
     // Drop unknown leaf frame (method_id == NULL at index 0).
     // Returns the new depth after removal.
     static inline int dropUnknownLeaf(ASGCT_CallFrame* frames, int depth) {