From b507f77a631e5d6bd3eb55e8af864a9f30fcde33 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 08:55:56 -0400 Subject: [PATCH 01/26] PROF-16213: Account CTimer timer table in NativeMem CTimer::start() allocated its tid-indexed timer table (pid_max ints) without recording it, so it was missing from the native_mem_* counters. - Move the table sizing into CTimer::resizeTimers() and record the table under a new NM_CTIMER ("ctimer") category, mirroring how PerfEvents accounts its per-thread array under NM_PERF. - On calloc failure, leave the table empty (_max_timers = 0) instead of a NULL table with a non-zero bound, and fail start() with "Failed to allocate CPU timer table". The tid bound checks then reject every tid, and the next start() retries the allocation. - Add ctimer_ut covering accounting across resizes, same-size restart, and the allocation-failure path. - Document NM_CTIMER in ProfilingEngines.md. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer.h | 4 + ddprof-lib/src/main/cpp/ctimer_linux.cpp | 32 +++++-- ddprof-lib/src/main/cpp/nativeMem.h | 1 + ddprof-lib/src/test/cpp/ctimer_ut.cpp | 104 +++++++++++++++++++++++ doc/architecture/ProfilingEngines.md | 7 +- 5 files changed, 140 insertions(+), 8 deletions(-) create mode 100644 ddprof-lib/src/test/cpp/ctimer_ut.cpp diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h index cfc79414ac..01570a2e99 100644 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ b/ddprof-lib/src/main/cpp/ctimer.h @@ -1,5 +1,6 @@ /* * Copyright 2017 Andrei Pangin + * Copyright 2026, Datadog, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +26,8 @@ #include class CTimer : public Engine { + friend class CTimerTestAccessor; + private: // cppcheck-suppress unusedPrivateFunction static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); @@ -41,6 +44,7 @@ class CTimer : public Engine { static int _max_timers; static int *_timers; + static bool resizeTimers(int max_timers); int registerThread(int tid); void unregisterThread(int tid); diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index f1e1a8a418..7e86ba2c0c 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -24,6 +24,7 @@ #include "debugSupport.h" #include "libraries.h" #include "log.h" +#include "nativeMem.h" #include "profiler.h" #include "samplerPerf.h" #include "signalCookie.h" @@ -131,6 +132,30 @@ void CTimer::unregisterThread(int tid) { } } +// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. On +// allocation failure the table is left empty (_max_timers == 0), so the +// tid bound checks in registerThread()/unregisterThread()/stop() reject every +// tid instead of indexing a NULL table, and the next start() retries. +bool CTimer::resizeTimers(int max_timers) { + if (max_timers == _max_timers) { + return true; + } + long long old_bytes = + (_timers != NULL) ? (long long)((size_t)_max_timers * sizeof(int)) : 0; + free(_timers); + if (old_bytes > 0) { + NativeMem::record(NM_CTIMER, -old_bytes); + } + _timers = (int *)calloc(max_timers, sizeof(int)); + if (_timers == NULL) { + _max_timers = 0; + return false; + } + _max_timers = max_timers; + NativeMem::record(NM_CTIMER, (long long)((size_t)max_timers * sizeof(int))); + return true; +} + Error CTimer::check(Arguments &args) { timer_t timer; if (timer_create(CLOCK_THREAD_CPUTIME_ID, NULL, &timer) < 0) { @@ -150,11 +175,8 @@ Error CTimer::start(Arguments &args) { _cstack = args._cstack; _signal = SIGPROF; - int max_timers = OS::getMaxThreadId(); - if (max_timers != _max_timers) { - free(_timers); - _timers = (int *)calloc(max_timers, sizeof(int)); - _max_timers = max_timers; + if (!resizeTimers(OS::getMaxThreadId())) { + return Error("Failed to allocate CPU timer table"); } // Prime the origin-check cache from this non-signal context before any diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index cbcb70844d..8545e1dd1c 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -38,6 +38,7 @@ X(METHOD_MAP, "method_map") \ X(LIVENESS, "liveness") \ X(WALLCLOCK, "wallclock") \ + X(CTIMER, "ctimer") \ X(MISC, "misc") #define X_NM_ENUM(a, b) NM_##a, diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp new file mode 100644 index 0000000000..00a7d3a30f --- /dev/null +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -0,0 +1,104 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include + +#ifdef __linux__ + +#include "ctimer.h" +#include "nativeMem.h" + +#include + +// Exposes CTimer's tid-indexed timer table without starting the engine, which +// would install a SIGPROF handler and arm a timer for every live thread. +class CTimerTestAccessor { +public: + static bool resizeTimers(int max_timers) { return CTimer::resizeTimers(max_timers); } + static int maxTimers() { return CTimer::_max_timers; } + static int *timers() { return CTimer::_timers; } + + // Hands the current table to the caller without freeing it, so a test can + // run against an empty table and later reinstate the original one. + static void detach() { + CTimer::_timers = nullptr; + CTimer::_max_timers = 0; + } + + // Frees the test-owned table and reinstates the detached one, without + // touching the gauge (the fixture restores that separately). + static void reattach(int max_timers, int *timers) { + free(CTimer::_timers); + CTimer::_timers = timers; + CTimer::_max_timers = max_timers; + } +}; + +class CTimerNativeMemTest : public ::testing::Test { +protected: + int _saved_max; + int *_saved_timers; + long long _saved_live; + + void SetUp() override { + _saved_max = CTimerTestAccessor::maxTimers(); + _saved_timers = CTimerTestAccessor::timers(); + _saved_live = NativeMem::live(NM_CTIMER); + // Detach the existing table so resizeTimers() neither frees it nor + // subtracts its size; the gauge starts from zero for this test. + CTimerTestAccessor::detach(); + NativeMem::setLive(NM_CTIMER, 0); + } + + void TearDown() override { + CTimerTestAccessor::reattach(_saved_max, _saved_timers); + NativeMem::setLive(NM_CTIMER, _saved_live); + } +}; + +// The timer table is reported under NM_CTIMER, and a resize replaces the old +// size rather than adding to it. +TEST_F(CTimerNativeMemTest, TimerTableIsAccounted) { + CTimerTestAccessor::resizeTimers(1024); + ASSERT_NE(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); + + CTimerTestAccessor::resizeTimers(4096); + ASSERT_NE(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ((long long)(4096 * sizeof(int)), NativeMem::live(NM_CTIMER)); + + CTimerTestAccessor::resizeTimers(512); + EXPECT_EQ((long long)(512 * sizeof(int)), NativeMem::live(NM_CTIMER)); +} + +// Restarting with an unchanged pid_max keeps the table and the gauge as-is. +TEST_F(CTimerNativeMemTest, SameSizeDoesNotReaccount) { + CTimerTestAccessor::resizeTimers(2048); + int *table = CTimerTestAccessor::timers(); + + CTimerTestAccessor::resizeTimers(2048); + EXPECT_EQ(table, CTimerTestAccessor::timers()); + EXPECT_EQ((long long)(2048 * sizeof(int)), NativeMem::live(NM_CTIMER)); +} + +// A failed allocation leaves an empty table rather than a NULL table with a +// non-zero bound, releases the old table's bytes from the gauge, and a later +// resize to a valid size succeeds. A negative count makes calloc's +// count * size overflow, so the allocation fails deterministically. +TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(1024)); + + EXPECT_FALSE(CTimerTestAccessor::resizeTimers(-1)); + EXPECT_EQ(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ(0, CTimerTestAccessor::maxTimers()); + EXPECT_EQ(0, NativeMem::live(NM_CTIMER)); + + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(1024)); + EXPECT_NE(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ(1024, CTimerTestAccessor::maxTimers()); + EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); +} + +#endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index e40fc6f6df..11d10f25b9 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -459,9 +459,9 @@ In summary: ## Profiler Self-Accounting (`NativeMem`, `nativeMem.{h,cpp}`) `nativeMem.{h,cpp}` is **not** a profiling engine and does not observe -application allocations. It tracks the profiler's **own** native memory in 13 +application allocations. It tracks the profiler's **own** native memory in 14 categories (`NM_CALLTRACE`, `NM_DICTIONARY`, ..., `NM_PERF`, `NM_LIVENESS`, -`NM_WALLCLOCK`, `NM_MISC`; `nativeMem.h`). +`NM_WALLCLOCK`, `NM_CTIMER`, `NM_MISC`; `nativeMem.h`). - `record(cat, delta)` is a relaxed atomic add plus a lock-free CAS on the per-category peak. It is async-signal-safe, which matters because the @@ -476,7 +476,8 @@ categories (`NM_CALLTRACE`, `NM_DICTIONARY`, ..., `NM_PERF`, `NM_LIVENESS`, exported as the `NATIVE_MEM_LIVE_BYTES`, `NATIVE_MEM_AVG_BYTES`, and `NATIVE_MEM_MAX_BYTES` counters (`flightRecorder.cpp:2201-2203`). -The engines above feed it: `PerfEvents` reports `NM_PERF`, +The engines above feed it: `PerfEvents` reports `NM_PERF`, `CTimer` reports +its tid-indexed timer table (`pid_max × sizeof(int)`) as `NM_CTIMER`, `timerLoopCommon()` reports `NM_WALLCLOCK`, and `LivenessTracker` reports `NM_LIVENESS`. From d727893c3ce3b6aaaf033ac8ac1d522c6aeaedaa Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 09:21:48 -0400 Subject: [PATCH 02/26] PROF-16213: Harden ctimer_ut failure-path coverage - Skip the calloc-overflow tests under ASan/TSan. The CI sanitizer jobs run gtest binaries without *SAN_OPTIONS, so allocator_may_return_null defaults to 0 and the overflow is reported as a fatal error. - Route CTimer::start() through startWithTableSize() so a test can drive the table-allocation failure and check that start() returns an error before installing the handler or arming timers. - Check a sentinel survives a same-size resize, proving the table is not reallocated (calloc may return the same block after free). Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer.h | 3 ++ ddprof-lib/src/main/cpp/ctimer_linux.cpp | 6 +++- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 44 +++++++++++++++++++++--- 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h index 01570a2e99..3aac58372d 100644 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ b/ddprof-lib/src/main/cpp/ctimer.h @@ -45,6 +45,9 @@ class CTimer : public Engine { static int *_timers; static bool resizeTimers(int max_timers); + // start() with the timer table size passed in rather than read from + // pid_max, so a test can drive the allocation-failure path. + Error startWithTableSize(Arguments &args, int max_timers); int registerThread(int tid); void unregisterThread(int tid); diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 7e86ba2c0c..d87294c55e 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -167,6 +167,10 @@ Error CTimer::check(Arguments &args) { } Error CTimer::start(Arguments &args) { + return startWithTableSize(args, OS::getMaxThreadId()); +} + +Error CTimer::startWithTableSize(Arguments &args, int max_timers) { if (args._interval < 0) { return Error("interval must be positive"); } @@ -175,7 +179,7 @@ Error CTimer::start(Arguments &args) { _cstack = args._cstack; _signal = SIGPROF; - if (!resizeTimers(OS::getMaxThreadId())) { + if (!resizeTimers(max_timers)) { return Error("Failed to allocate CPU timer table"); } diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 00a7d3a30f..500cb2aa57 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -7,11 +7,24 @@ #ifdef __linux__ +#include "arguments.h" +#include "common.h" #include "ctimer.h" #include "nativeMem.h" #include +// The failure-path tests force calloc to fail with a negative count, which +// makes count * size overflow. The CI sanitizer jobs run gtest binaries +// without *SAN_OPTIONS, so allocator_may_return_null defaults to 0 and the +// sanitizer reports that overflow as a fatal error instead of returning NULL. +#if defined(ASAN_ENABLED) || defined(TSAN_ENABLED) +#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() \ + GTEST_SKIP() << "sanitizer allocators abort on calloc overflow" +#else +#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() +#endif + // Exposes CTimer's tid-indexed timer table without starting the engine, which // would install a SIGPROF handler and arm a timer for every live thread. class CTimerTestAccessor { @@ -19,6 +32,10 @@ class CTimerTestAccessor { static bool resizeTimers(int max_timers) { return CTimer::resizeTimers(max_timers); } static int maxTimers() { return CTimer::_max_timers; } static int *timers() { return CTimer::_timers; } + static Error startWithTableSize(CTimer &engine, Arguments &args, + int max_timers) { + return engine.startWithTableSize(args, max_timers); + } // Hands the current table to the caller without freeing it, so a test can // run against an empty table and later reinstate the original one. @@ -74,20 +91,24 @@ TEST_F(CTimerNativeMemTest, TimerTableIsAccounted) { } // Restarting with an unchanged pid_max keeps the table and the gauge as-is. +// The sentinel shows the table was not reallocated: a free + calloc of the +// same size can return the same block, but calloc zeroes it. TEST_F(CTimerNativeMemTest, SameSizeDoesNotReaccount) { - CTimerTestAccessor::resizeTimers(2048); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(2048)); int *table = CTimerTestAccessor::timers(); + table[2047] = 0x5a5a5a5a; - CTimerTestAccessor::resizeTimers(2048); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(2048)); EXPECT_EQ(table, CTimerTestAccessor::timers()); + EXPECT_EQ(0x5a5a5a5a, CTimerTestAccessor::timers()[2047]); EXPECT_EQ((long long)(2048 * sizeof(int)), NativeMem::live(NM_CTIMER)); } // A failed allocation leaves an empty table rather than a NULL table with a // non-zero bound, releases the old table's bytes from the gauge, and a later -// resize to a valid size succeeds. A negative count makes calloc's -// count * size overflow, so the allocation fails deterministically. +// resize to a valid size succeeds. TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { + SKIP_IF_CALLOC_OVERFLOW_ABORTS(); ASSERT_TRUE(CTimerTestAccessor::resizeTimers(1024)); EXPECT_FALSE(CTimerTestAccessor::resizeTimers(-1)); @@ -101,4 +122,19 @@ TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); } +// start() fails when the timer table cannot be allocated, and returns before +// it installs the SIGPROF handler or arms a timer for any thread. +TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { + SKIP_IF_CALLOC_OVERFLOW_ABORTS(); + CTimer engine; + Arguments args; + + Error error = CTimerTestAccessor::startWithTableSize(engine, args, -1); + EXPECT_TRUE((bool)error); + EXPECT_STREQ("Failed to allocate CPU timer table", error.message()); + EXPECT_EQ(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ(0, CTimerTestAccessor::maxTimers()); + EXPECT_EQ(0, NativeMem::live(NM_CTIMER)); +} + #endif // __linux__ From 83d5e2120daa30985371ba4d062ee9afd378c282 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 09:23:49 -0400 Subject: [PATCH 03/26] PROF-16213: Clarify where NativeMem categories are defined Co-Authored-By: Claude Opus 5.5 --- doc/architecture/ProfilingEngines.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 11d10f25b9..7729560e87 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -460,8 +460,8 @@ In summary: `nativeMem.{h,cpp}` is **not** a profiling engine and does not observe application allocations. It tracks the profiler's **own** native memory in 14 -categories (`NM_CALLTRACE`, `NM_DICTIONARY`, ..., `NM_PERF`, `NM_LIVENESS`, -`NM_WALLCLOCK`, `NM_CTIMER`, `NM_MISC`; `nativeMem.h`). +categories, defined in `nativeMem.h`: `NM_CALLTRACE`, `NM_DICTIONARY`, ..., +`NM_PERF`, `NM_LIVENESS`, `NM_WALLCLOCK`, `NM_CTIMER`, `NM_MISC`. - `record(cat, delta)` is a relaxed atomic add plus a lock-free CAS on the per-category peak. It is async-signal-safe, which matters because the From 0d93361714e276f7dbca4785c779d703635d5424 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 10:49:52 -0400 Subject: [PATCH 04/26] PROF-16213: Keep the native-thread hook off engines during start/stop Profiler::start() installs the pthread_create hook before it starts the engines, and the hook calls Profiler::registerThread()/unregisterThread() without _state_lock. A thread created in that window reached CTimer::registerThread() while resizeTimers() freed and replaced the timer table, with no lock or ordering between them. Add Profiler::_engines_ready. start() sets it (release) once the engines have started; stop() clears it after the signal drain and before any engine stops. registerThread()/unregisterThread() return early unless it is set (acquire). Engine start() registers existing threads and engine stop() unregisters all of them, so skipped calls neither miss nor leak timers outside the documented start window. Add a gtest showing registerThread() does not touch the (null) engines while the flag is clear, and update ProfilingEngines.md for the gate and shifted profiler.cpp line references. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 11 +++--- ddprof-lib/src/main/cpp/profiler.cpp | 18 ++++++++++ ddprof-lib/src/main/cpp/profiler.h | 9 ++++- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 20 +++++++++++ doc/architecture/ProfilingEngines.md | 43 ++++++++++++++---------- 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index d87294c55e..d4138bdb8f 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -132,10 +132,13 @@ void CTimer::unregisterThread(int tid) { } } -// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. On -// allocation failure the table is left empty (_max_timers == 0), so the -// tid bound checks in registerThread()/unregisterThread()/stop() reject every -// tid instead of indexing a NULL table, and the next start() retries. +// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs from +// start() before the profiler sets its engines-ready flag, so the pthread hook's +// Profiler::registerThread()/unregisterThread() cannot touch the table while it +// is freed and replaced, and JVMTI thread events are not enabled yet. On +// allocation failure the table is left empty (_max_timers == 0), so the tid +// bound checks reject every tid instead of indexing a NULL table, and the next +// start() retries. bool CTimer::resizeTimers(int max_timers) { if (max_timers == _max_timers) { return true; diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 94f8df8d0d..0ce885ef28 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -165,6 +165,11 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { } int Profiler::registerThread(int tid) { + // Threads created while the engines are not started are picked up by the + // engines' own start(), which registers every existing thread. + if (!_instance->_engines_ready.load(std::memory_order_acquire)) { + return 0; + } return _instance->_cpu_engine->registerThread(tid) | _instance->_wall_engine->registerThread(tid); } @@ -190,6 +195,11 @@ void Profiler::unregisterThread(int tid) { g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); return; #endif + // Engine stop() unregisters every remaining thread, so skipping here while + // the engines are not started does not leak a per-thread timer. + if (!_instance->_engines_ready.load(std::memory_order_acquire)) { + return; + } _instance->_cpu_engine->unregisterThread(tid); _instance->_wall_engine->unregisterThread(tid); } @@ -1761,6 +1771,10 @@ Error Profiler::start(Arguments &args, bool reset) { } if (activated) { + // Release pairs with the acquire in registerThread()/unregisterThread(): + // a hook that sees true also sees the per-thread structures the engine + // start() calls above have finished sizing. + _engines_ready.store(true, std::memory_order_release); switchThreadEvents(JVMTI_ENABLE); if (_reference_chains_active) { @@ -1830,6 +1844,10 @@ Error Profiler::stop() { return Error("signal handlers did not drain; teardown skipped, retry stop()"); } + // Keep the pthread hook off the engines before any engine stop() runs, so + // no thread arms a timer after CTimer::stop() has unregistered them all. + _engines_ready.store(false, std::memory_order_release); + if (_event_mask & EM_ALLOC) _alloc_engine->stop(); if (_reference_chains_active) { diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 6c85387d0a..84ca7bfc21 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -89,6 +89,12 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; + // True while the CPU/wall engines are started and their per-thread + // structures are stable. registerThread()/unregisterThread() are reached + // from the pthread hook without _state_lock, so they check this flag to + // stay off the engines while start() sizes those structures (for example + // CTimer::resizeTimers()) and while stop() tears them down. + std::atomic _engines_ready; // class unload hook Trap _class_unload_hook_trap; typedef void (*NotifyClassUnloadedFunc)(void *); @@ -249,7 +255,8 @@ class alignas(alignof(SpinLock)) Profiler { public: Profiler() - : _state_lock(), _state(State::NEW), _class_unload_hook_trap(2), + : _state_lock(), _state(State::NEW), _engines_ready(false), + _class_unload_hook_trap(2), _notify_class_unloaded_func(NULL), _thread_info(), _class_map(1), _string_label_map(2), _context_value_map(3), _thread_filter(), _call_trace_storage(), _jfr(), _cpu_engine(NULL), _wall_engine(NULL), diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 500cb2aa57..1b80c0e8e7 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -11,6 +11,8 @@ #include "common.h" #include "ctimer.h" #include "nativeMem.h" +#include "os.h" +#include "profiler.h" #include @@ -137,4 +139,22 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { EXPECT_EQ(0, NativeMem::live(NM_CTIMER)); } +class ProfilerTestAccessor { +public: + static bool enginesReady(Profiler *p) { + return p->_engines_ready.load(std::memory_order_acquire); + } +}; + +// Profiler::registerThread() is reached from the pthread hook without +// _state_lock. Until start() has sized the engines' per-thread structures +// (CTimer::resizeTimers()) and set the engines-ready flag, it must return +// without touching the engines. In this binary the profiler is never +// started, so its engine pointers are null and any engine call would crash. +TEST(ProfilerEnginesGateTest, RegisterThreadSkipsEnginesUntilReady) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + EXPECT_EQ(0, Profiler::registerThread(OS::threadId())); +} + #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 7729560e87..0579033e26 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1269`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1316`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1351`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1279`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1326`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1361`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,7 +65,7 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1453 +Profiler::start() profiler.cpp:1463 selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire @@ -76,30 +76,33 @@ Profiler::start() profiler.cpp:1453 malloc_tracer.start() (EM_NATIVEMEM) fatal only if nativemem is the sole mode NativeSocketSampler::start() (EM_NATIVESOCKET) if anything activated: + _engines_ready = true gates Profiler::registerThread() switchThreadEvents(JVMTI_ENABLE) _cpu_engine->enableEvents(true) LAST ``` - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1690-1696`). -- The CPU engine is enabled last (`profiler.cpp:1791`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1700-1706`). +- The CPU engine is enabled last (`profiler.cpp:1805`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1811`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1825`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1004`). + only (`profiler.cpp:1014`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1824`). -3. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, + and tears nothing down (`profiler.cpp:1838`). +3. `_engines_ready` is cleared (`profiler.cpp:1849`), so the native-thread + hook stops calling into the engines before any of them stops. +4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:1957`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:1975`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -116,7 +119,11 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: wraps the start routine (`start_routine_wrapper()`) and calls `Profiler::registerThread()` (`libraryPatcher_linux.cpp:553`). On exit, `unregister_and_release()` calls `Profiler::unregisterThread()` - (`libraryPatcher_linux.cpp:249`). + (`libraryPatcher_linux.cpp:249`). The hook is installed before the engines + start and runs without `_state_lock`, so both calls return without touching + the engines unless `_engines_ready` is set (`profiler.cpp:1777`). This keeps + them off per-thread structures that `start()` is still sizing, such as + `CTimer`'s tid-indexed timer table, and that `stop()` is tearing down. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:980`). @@ -143,7 +150,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:680-691`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:690-701`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -154,7 +161,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1269-1314`): +`selectCpuEngine()` (`profiler.cpp:1279-1324`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -234,7 +241,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1622`, `1642`). + engine (`profiler.cpp:1632`, `1652`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -269,7 +276,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1316-1349`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1326-1359`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -340,7 +347,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:328` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:338` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -409,7 +416,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2035`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2053`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From 54689c5350375fb8d57d40b9ee7df113843a62ac Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 11:24:20 -0400 Subject: [PATCH 05/26] PROF-16213: Drain in-flight hook calls and open the gate after CPU start The _engines_ready check in Profiler::registerThread()/unregisterThread() was a single load. A hook call that passed it just before stop() cleared the flag could still arm a CTimer timer after CTimer::stop() unregistered all slots, or touch the timer table while the next start() resized it. - Count each hook call in flight with an RAII HookCallScope. stop() clears the flag and waits for the count to reach zero before any engine stops. Flag and counter use seq_cst on both sides, so either the hook sees the flag cleared or stop() sees the call and waits. - Set _engines_ready as soon as the CPU engine has started instead of after all engines. The wall engines do not implement registerThread(), and the earlier set narrows the start window in which a native thread that starts or exits is missed to the rest of CTimer::start(). - Check the in-flight count returns to zero in the gate test, and update ProfilingEngines.md for the drain, the new set point, and shifted profiler.cpp line references. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/profiler.cpp | 54 +++++++++++++++++++++------ ddprof-lib/src/main/cpp/profiler.h | 14 ++++--- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 6 +++ doc/architecture/ProfilingEngines.md | 50 ++++++++++++++----------- 4 files changed, 85 insertions(+), 39 deletions(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 0ce885ef28..430fe411ee 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -164,10 +165,30 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { LivenessTracker::instance()->releaseThreadLocalState(); } +// Marks a native-thread hook call as in flight for the duration of a +// Profiler::registerThread()/unregisterThread() call, so stop() can wait for +// calls that passed the _engines_ready check before it stops the engines. +// The increment and the flag load in the caller are seq_cst, pairing with the +// seq_cst flag store and counter load in stop(): either the hook sees the flag +// cleared, or stop() sees the call in flight and waits for it. +class HookCallScope { +public: + explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { + _inflight.fetch_add(1, std::memory_order_seq_cst); + } + ~HookCallScope() { _inflight.fetch_sub(1, std::memory_order_release); } + HookCallScope(const HookCallScope &) = delete; + HookCallScope &operator=(const HookCallScope &) = delete; + +private: + std::atomic &_inflight; +}; + int Profiler::registerThread(int tid) { + HookCallScope scope(_instance->_hook_calls_inflight); // Threads created while the engines are not started are picked up by the - // engines' own start(), which registers every existing thread. - if (!_instance->_engines_ready.load(std::memory_order_acquire)) { + // CPU engine's own start(), which registers every existing thread. + if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return 0; } return _instance->_cpu_engine->registerThread(tid) | @@ -195,9 +216,11 @@ void Profiler::unregisterThread(int tid) { g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); return; #endif - // Engine stop() unregisters every remaining thread, so skipping here while - // the engines are not started does not leak a per-thread timer. - if (!_instance->_engines_ready.load(std::memory_order_acquire)) { + HookCallScope scope(_instance->_hook_calls_inflight); + // Outside the ready window the CPU engine either has not registered any + // thread yet or is about to unregister all of them in stop(), so skipping + // here does not leave a timer behind. + if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return; } _instance->_cpu_engine->unregisterThread(tid); @@ -1695,6 +1718,12 @@ Error Profiler::start(Arguments &args, bool reset) { error = Error::OK; // recoverable } else { activated |= EM_CPU; + // Open the hook gate as soon as the CPU engine has sized its per-thread + // structures and registered the existing threads, so native threads + // that start or exit from here on are registered or unregistered. + // The wall engines do not implement registerThread(), so they do not + // need to have started yet. + _engines_ready.store(true, std::memory_order_seq_cst); } } if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { @@ -1771,10 +1800,6 @@ Error Profiler::start(Arguments &args, bool reset) { } if (activated) { - // Release pairs with the acquire in registerThread()/unregisterThread(): - // a hook that sees true also sees the per-thread structures the engine - // start() calls above have finished sizing. - _engines_ready.store(true, std::memory_order_release); switchThreadEvents(JVMTI_ENABLE); if (_reference_chains_active) { @@ -1844,9 +1869,14 @@ Error Profiler::stop() { return Error("signal handlers did not drain; teardown skipped, retry stop()"); } - // Keep the pthread hook off the engines before any engine stop() runs, so - // no thread arms a timer after CTimer::stop() has unregistered them all. - _engines_ready.store(false, std::memory_order_release); + // Close the hook gate and wait for hook calls already past it, so no thread + // arms a timer after CTimer::stop() has unregistered them all, and none is + // still inside the timer table when the next start() resizes it. Hook calls + // only do a few syscalls per engine, so the wait is short. + _engines_ready.store(false, std::memory_order_seq_cst); + while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { + sched_yield(); + } if (_event_mask & EM_ALLOC) _alloc_engine->stop(); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 84ca7bfc21..82c4bcdb9b 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -89,12 +89,16 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; - // True while the CPU/wall engines are started and their per-thread - // structures are stable. registerThread()/unregisterThread() are reached + // True from the moment the CPU engine has started until stop() begins + // tearing the engines down. registerThread()/unregisterThread() are reached // from the pthread hook without _state_lock, so they check this flag to - // stay off the engines while start() sizes those structures (for example - // CTimer::resizeTimers()) and while stop() tears them down. + // stay off the engines while start() sizes their per-thread structures (for + // example CTimer::resizeTimers()) and while stop() tears them down. std::atomic _engines_ready; + // Hook calls that are past their _engines_ready check and may still be + // inside an engine. stop() waits for this to reach zero after clearing + // _engines_ready, before any engine stops. + std::atomic _hook_calls_inflight; // class unload hook Trap _class_unload_hook_trap; typedef void (*NotifyClassUnloadedFunc)(void *); @@ -256,7 +260,7 @@ class alignas(alignof(SpinLock)) Profiler { public: Profiler() : _state_lock(), _state(State::NEW), _engines_ready(false), - _class_unload_hook_trap(2), + _hook_calls_inflight(0), _class_unload_hook_trap(2), _notify_class_unloaded_func(NULL), _thread_info(), _class_map(1), _string_label_map(2), _context_value_map(3), _thread_filter(), _call_trace_storage(), _jfr(), _cpu_engine(NULL), _wall_engine(NULL), diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 1b80c0e8e7..f752fc83f2 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -144,6 +144,9 @@ class ProfilerTestAccessor { static bool enginesReady(Profiler *p) { return p->_engines_ready.load(std::memory_order_acquire); } + static int hookCallsInflight(Profiler *p) { + return p->_hook_calls_inflight.load(std::memory_order_acquire); + } }; // Profiler::registerThread() is reached from the pthread hook without @@ -151,10 +154,13 @@ class ProfilerTestAccessor { // (CTimer::resizeTimers()) and set the engines-ready flag, it must return // without touching the engines. In this binary the profiler is never // started, so its engine pointers are null and any engine call would crash. +// The call must also leave the in-flight count at zero, or stop() would wait +// on it forever. TEST(ProfilerEnginesGateTest, RegisterThreadSkipsEnginesUntilReady) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); EXPECT_EQ(0, Profiler::registerThread(OS::threadId())); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 0579033e26..81d3ee9354 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1279`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1326`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1361`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1302`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1349`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1384`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,44 +65,46 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1463 +Profiler::start() profiler.cpp:1486 selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire _cpu_engine->start() (EM_CPU) failure -> warning, continue + on success: _engines_ready = true opens Profiler::registerThread() _wall_engine->enableEvents(true) BEFORE start(): see below _wall_engine->start() (EM_WALL) _alloc_engine = selectAllocEngine(); start() (EM_ALLOC) malloc_tracer.start() (EM_NATIVEMEM) fatal only if nativemem is the sole mode NativeSocketSampler::start() (EM_NATIVESOCKET) if anything activated: - _engines_ready = true gates Profiler::registerThread() switchThreadEvents(JVMTI_ENABLE) _cpu_engine->enableEvents(true) LAST ``` - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1700-1706`). -- The CPU engine is enabled last (`profiler.cpp:1805`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1729-1735`). +- The CPU engine is enabled last (`profiler.cpp:1830`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1825`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1850`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1014`). + only (`profiler.cpp:1037`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1838`). -3. `_engines_ready` is cleared (`profiler.cpp:1849`), so the native-thread - hook stops calling into the engines before any of them stops. + and tears nothing down (`profiler.cpp:1863`). +3. `_engines_ready` is cleared, and `stop()` waits until no hook call that + passed the check is still inside an engine (`profiler.cpp:1876-1879`). + No native thread can then arm a timer after `CTimer::stop()`, or still be + using the timer table when the next `start()` resizes it. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:1975`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2005`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -121,9 +123,13 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: `unregister_and_release()` calls `Profiler::unregisterThread()` (`libraryPatcher_linux.cpp:249`). The hook is installed before the engines start and runs without `_state_lock`, so both calls return without touching - the engines unless `_engines_ready` is set (`profiler.cpp:1777`). This keeps - them off per-thread structures that `start()` is still sizing, such as - `CTimer`'s tid-indexed timer table, and that `stop()` is tearing down. + the engines unless `_engines_ready` is set. `start()` sets it as soon as the + CPU engine has started (`profiler.cpp:1726`). This keeps the calls off + per-thread structures that `start()` is still sizing, such as `CTimer`'s + tid-indexed timer table, and that `stop()` is tearing down. A native thread + that starts or exits after the CPU engine lists threads, but before the flag + is set, is missed: it is not registered, or its timer slot is left in place + until `stop()`. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:980`). @@ -150,7 +156,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:690-701`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:713-724`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -161,7 +167,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1279-1324`): +`selectCpuEngine()` (`profiler.cpp:1302-1347`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -241,7 +247,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1632`, `1652`). + engine (`profiler.cpp:1655`, `1675`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -276,7 +282,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1326-1359`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1349-1382`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -347,7 +353,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:338` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:361` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -416,7 +422,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2053`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2083`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From 9d581c56bb56ddab346b510fb353a8466d193901 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 12:28:33 -0400 Subject: [PATCH 06/26] PROF-16213: Open thread registration before engines list threads The hook gate opened only after the CPU engine's start() returned, so native-thread hook calls made while start() registered the existing threads were dropped: a thread starting after the thread list was taken was never registered, and one exiting after being registered left a timer slot that blocked a reused tid for the whole recording. - Add Profiler::openThreadRegistration()/closeThreadRegistration(). CTimer::start() and PerfEvents::start() open the gate once their per-thread table is sized and before they list threads; concurrent registration of one tid by the hook and by start() is resolved by the existing per-slot CAS. Profiler::start() still opens it after a successful CPU engine start, for engines that do not, and closes it (with the in-flight wait) when the CPU engine fails. stop() uses closeThreadRegistration(). - Add a gtest that a successful CTimer start opens the gate, and check that a failed one leaves it closed. - Update ProfilingEngines.md for the new open point, and fix profiler.cpp, ctimer_linux.cpp and perfEvents_linux.cpp line references, including ctimer_linux.cpp ones left stale earlier in this branch. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 12 +++- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 5 ++ ddprof-lib/src/main/cpp/profiler.cpp | 42 +++++++----- ddprof-lib/src/main/cpp/profiler.h | 19 ++++-- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 44 +++++++++--- doc/architecture/ProfilingEngines.md | 72 ++++++++++---------- 6 files changed, 125 insertions(+), 69 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index d4138bdb8f..112c2af5dc 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -133,9 +133,9 @@ void CTimer::unregisterThread(int tid) { } // Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs from -// start() before the profiler sets its engines-ready flag, so the pthread hook's -// Profiler::registerThread()/unregisterThread() cannot touch the table while it -// is freed and replaced, and JVMTI thread events are not enabled yet. On +// start() before it calls Profiler::openThreadRegistration(), so the pthread +// hook's Profiler::registerThread()/unregisterThread() cannot touch the table +// while it is freed and replaced, and JVMTI thread events are not enabled yet. On // allocation failure the table is left empty (_max_timers == 0), so the tid // bound checks reject every tid instead of indexing a NULL table, and the next // start() retries. @@ -194,6 +194,12 @@ Error CTimer::startWithTableSize(Arguments &args, int max_timers) { OS::installSignalHandler(_signal, signalHandler); + // The table is sized, so let the native-thread hook register and unregister + // threads from here on. Opening the gate before listing threads means a + // thread that starts after the list is taken is registered by the hook, and + // one that exits after being registered below is unregistered by it. + Profiler::instance()->openThreadRegistration(); + // Register all existing threads. Individual failures are benign — a thread // may exit between listThreads() and registerThread(), and new threads // will register themselves on creation. check() already validated that the diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 05c1e82454..451a917acc 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -980,6 +980,11 @@ Error PerfEvents::start(Arguments &args) { __atomic_store_n(_pthread_entry, (void *)pthread_setspecific_hook, __ATOMIC_RELEASE); + // _events is sized, so let the native-thread hook register and unregister + // threads from here on; opening the gate before listing threads means no + // thread that starts or exits during the loop below is missed. + Profiler::instance()->openThreadRegistration(); + // Create perf_events for all existing threads int threads_sz = 0, threads_cap; int *threads = (int *)malloc((threads_cap = 1024) * sizeof(int)); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 430fe411ee..bed7048b37 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -184,10 +184,22 @@ class HookCallScope { std::atomic &_inflight; }; +void Profiler::openThreadRegistration() { + _engines_ready.store(true, std::memory_order_seq_cst); +} + +void Profiler::closeThreadRegistration() { + // Hook calls only do a few syscalls per engine, so the wait is short. + _engines_ready.store(false, std::memory_order_seq_cst); + while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { + sched_yield(); + } +} + int Profiler::registerThread(int tid) { HookCallScope scope(_instance->_hook_calls_inflight); - // Threads created while the engines are not started are picked up by the - // CPU engine's own start(), which registers every existing thread. + // Threads created before the gate opens are picked up by the CPU engine's + // start(), which opens the gate before it lists the existing threads. if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return 0; } @@ -217,9 +229,9 @@ void Profiler::unregisterThread(int tid) { return; #endif HookCallScope scope(_instance->_hook_calls_inflight); - // Outside the ready window the CPU engine either has not registered any - // thread yet or is about to unregister all of them in stop(), so skipping - // here does not leave a timer behind. + // Outside the open gate the CPU engine either has not registered any thread + // yet (it opens the gate before listing threads) or stop() is about to + // unregister all of them, so skipping here does not leave a timer behind. if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return; } @@ -1714,16 +1726,16 @@ Error Profiler::start(Arguments &args, bool reset) { if ((_event_mask & EM_CPU) && _cpu_engine != &noop_engine) { error = _cpu_engine->start(args); if (error) { + // The engine may have opened the gate before failing. + closeThreadRegistration(); Log::warn("%s", error.message()); error = Error::OK; // recoverable } else { activated |= EM_CPU; - // Open the hook gate as soon as the CPU engine has sized its per-thread - // structures and registered the existing threads, so native threads - // that start or exit from here on are registered or unregistered. - // The wall engines do not implement registerThread(), so they do not - // need to have started yet. - _engines_ready.store(true, std::memory_order_seq_cst); + // CTimer and PerfEvents already opened the gate inside start(); this + // covers CPU engines that do not. The wall engines do not implement + // registerThread(), so they do not need to have started yet. + openThreadRegistration(); } } if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { @@ -1871,12 +1883,8 @@ Error Profiler::stop() { // Close the hook gate and wait for hook calls already past it, so no thread // arms a timer after CTimer::stop() has unregistered them all, and none is - // still inside the timer table when the next start() resizes it. Hook calls - // only do a few syscalls per engine, so the wait is short. - _engines_ready.store(false, std::memory_order_seq_cst); - while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { - sched_yield(); - } + // still inside the timer table when the next start() resizes it. + closeThreadRegistration(); if (_event_mask & EM_ALLOC) _alloc_engine->stop(); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 82c4bcdb9b..991926f69e 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -89,15 +89,13 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; - // True from the moment the CPU engine has started until stop() begins - // tearing the engines down. registerThread()/unregisterThread() are reached + // Set by openThreadRegistration() and cleared by closeThreadRegistration(). registerThread()/unregisterThread() are reached // from the pthread hook without _state_lock, so they check this flag to // stay off the engines while start() sizes their per-thread structures (for // example CTimer::resizeTimers()) and while stop() tears them down. std::atomic _engines_ready; // Hook calls that are past their _engines_ready check and may still be - // inside an engine. stop() waits for this to reach zero after clearing - // _engines_ready, before any engine stops. + // inside an engine. closeThreadRegistration() waits for this to reach zero. std::atomic _hook_calls_inflight; // class unload hook Trap _class_unload_hook_trap; @@ -385,6 +383,15 @@ class alignas(alignof(SpinLock)) Profiler { Error checkState(); void logStats(); void switchThreadEvents(jvmtiEventMode mode); + /** + * Lets the native-thread hook's registerThread()/unregisterThread() reach + * the engines. A CPU engine with per-thread structures calls this from its + * start() once those structures are sized and before it lists the existing + * threads, so no thread that starts or exits during start() is missed. + * start() also calls it after a successful CPU engine start, for engines + * that do not call it themselves. Idempotent. + */ + void openThreadRegistration(); /** * Remote symbolication packed data layout (BCI_NATIVE_FRAME_REMOTE): @@ -517,6 +524,10 @@ class alignas(alignof(SpinLock)) Profiler { static void setupSignalHandlers(); static void checkFault(ProfiledThread* thrd, siginfo_t *siginfo, void *ucontext); + // Closes the gate opened by openThreadRegistration() and waits for hook + // calls already past it to leave the engines. + void closeThreadRegistration(); + static int registerThread(int tid); static void unregisterThread(int tid); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index f752fc83f2..1f041b14b4 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -27,6 +27,19 @@ #define SKIP_IF_CALLOC_OVERFLOW_ABORTS() #endif +class ProfilerTestAccessor { +public: + static bool enginesReady(Profiler *p) { + return p->_engines_ready.load(std::memory_order_acquire); + } + static int hookCallsInflight(Profiler *p) { + return p->_hook_calls_inflight.load(std::memory_order_acquire); + } + static void closeThreadRegistration(Profiler *p) { + p->closeThreadRegistration(); + } +}; + // Exposes CTimer's tid-indexed timer table without starting the engine, which // would install a SIGPROF handler and arm a timer for every live thread. class CTimerTestAccessor { @@ -124,6 +137,26 @@ TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); } +// start() opens the native-thread registration gate once the table is sized, +// so a thread that starts or exits while start() registers the existing +// threads is handled by the pthread hook instead of being missed. The engine +// is stopped and the gate closed again before the fixture restores the table. +TEST_F(CTimerNativeMemTest, StartOpensThreadRegistration) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + CTimer engine; + Arguments args; + + Error error = CTimerTestAccessor::startWithTableSize(engine, args, + OS::getMaxThreadId()); + bool opened = ProfilerTestAccessor::enginesReady(profiler); + engine.stop(); + ProfilerTestAccessor::closeThreadRegistration(profiler); + + EXPECT_FALSE((bool)error); + EXPECT_TRUE(opened); +} + // start() fails when the timer table cannot be allocated, and returns before // it installs the SIGPROF handler or arms a timer for any thread. TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { @@ -133,22 +166,13 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { Error error = CTimerTestAccessor::startWithTableSize(engine, args, -1); EXPECT_TRUE((bool)error); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(Profiler::instance())); EXPECT_STREQ("Failed to allocate CPU timer table", error.message()); EXPECT_EQ(nullptr, CTimerTestAccessor::timers()); EXPECT_EQ(0, CTimerTestAccessor::maxTimers()); EXPECT_EQ(0, NativeMem::live(NM_CTIMER)); } -class ProfilerTestAccessor { -public: - static bool enginesReady(Profiler *p) { - return p->_engines_ready.load(std::memory_order_acquire); - } - static int hookCallsInflight(Profiler *p) { - return p->_hook_calls_inflight.load(std::memory_order_acquire); - } -}; - // Profiler::registerThread() is reached from the pthread hook without // _state_lock. Until start() has sized the engines' per-thread structures // (CTimer::resizeTimers()) and set the engines-ready flag, it must return diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 81d3ee9354..c09d9fb390 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1302`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1349`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1384`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1314`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1361`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1396`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,12 +65,13 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1486 +Profiler::start() profiler.cpp:1498 selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire _cpu_engine->start() (EM_CPU) failure -> warning, continue - on success: _engines_ready = true opens Profiler::registerThread() + CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads + on failure: closeThreadRegistration(); on success: openThreadRegistration() _wall_engine->enableEvents(true) BEFORE start(): see below _wall_engine->start() (EM_WALL) _alloc_engine = selectAllocEngine(); start() (EM_ALLOC) @@ -83,28 +84,28 @@ Profiler::start() profiler.cpp:1486 - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1729-1735`). -- The CPU engine is enabled last (`profiler.cpp:1830`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1741-1747`). +- The CPU engine is enabled last (`profiler.cpp:1842`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1850`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1862`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1037`). + only (`profiler.cpp:1049`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1863`). -3. `_engines_ready` is cleared, and `stop()` waits until no hook call that - passed the check is still inside an engine (`profiler.cpp:1876-1879`). + and tears nothing down (`profiler.cpp:1875`). +3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook + call that passed the check is still inside an engine (`profiler.cpp:1887`). No native thread can then arm a timer after `CTimer::stop()`, or still be using the timer table when the next `start()` resizes it. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2005`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2013`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -123,13 +124,14 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: `unregister_and_release()` calls `Profiler::unregisterThread()` (`libraryPatcher_linux.cpp:249`). The hook is installed before the engines start and runs without `_state_lock`, so both calls return without touching - the engines unless `_engines_ready` is set. `start()` sets it as soon as the - CPU engine has started (`profiler.cpp:1726`). This keeps the calls off - per-thread structures that `start()` is still sizing, such as `CTimer`'s - tid-indexed timer table, and that `stop()` is tearing down. A native thread - that starts or exits after the CPU engine lists threads, but before the flag - is set, is missed: it is not registered, or its timer slot is left in place - until `stop()`. + the engines unless `_engines_ready` is set. `CTimer::start()` and + `PerfEvents::start()` set it with `Profiler::openThreadRegistration()` once + their per-thread table is sized and before they list the existing threads + (`ctimer_linux.cpp:201`, `perfEvents_linux.cpp:986`). This keeps the calls + off a table that is still being freed and replaced, and that `stop()` is + tearing down, without missing a thread that starts or exits while `start()` + registers the existing ones. Concurrent registration of one tid by the hook + and by `start()` is resolved by the per-slot CAS. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:980`). @@ -156,7 +158,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:713-724`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:725-736`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -167,7 +169,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1302-1347`): +`selectCpuEngine()` (`profiler.cpp:1314-1359`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -203,24 +205,24 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Mechanism**: `registerThread(tid)` creates one POSIX timer per thread with the raw `__NR_timer_create` syscall. The clock is `((~tid) << 3) | 6`, the thread's per-thread scheduler CPU clock - (`ctimer_linux.cpp:45-47`). The raw syscall is used because the libc wrapper - accepts only predefined clocks (`ctimer_linux.cpp:85`). The timer notifies + (`ctimer_linux.cpp:46-48`). The raw syscall is used because the libc wrapper + accepts only predefined clocks (`ctimer_linux.cpp:86`). The timer notifies with `SIGEV_THREAD_ID` targeting `tid`, carrying - `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:72-83`). It is armed + `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:151`). +- **Signal**: `SIGPROF` (`ctimer_linux.cpp:183`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:261-265`). + (`ctimer_linux.cpp:296-300`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the - timer exactly once (`ctimer_linux.cpp:93-131`). A tid at or above + timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above `_max_timers` logs a warning and is not profiled. -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:181-187`). +- **`stop()`** unregisters every slot (`ctimer_linux.cpp:216-222`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:196-252`). It has no ASGCT fallback. + (`ctimer_linux.cpp:231-287`). It has no ASGCT fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -241,13 +243,13 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1047`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1052`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1655`, `1675`). + engine (`profiler.cpp:1667`, `1687`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -282,7 +284,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1349-1382`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1361-1394`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -353,7 +355,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:361` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:373` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -422,7 +424,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2083`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2091`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From 20f2d28af44d628ce08391e5c0c56c16ff98d8f1 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 14:12:35 -0400 Subject: [PATCH 07/26] PROF-16213: Keep PerfEvents unregistration open while idle; simplify gate Gating Profiler::unregisterThread() on _engines_ready skipped it after stop(). PerfEvents::stop() keeps per-thread fds open, so a thread started during a recording that exited after stop() leaked its perf fd and ring pages, and a thread reusing that tid at the next start was never sampled because registerThread() saw _fd > 0. - Give unregisterThread() its own gate, _thread_tables_stable. It is cleared only from CPU engine selection until the table is sized (RAII ThreadTablesFreeze over start()) and stays set while idle. Exits skipped in that window leave nothing behind: CTimer::stop() deleted every timer and PerfEvents::start() closes the fd of every tid missing from its thread list. - Drop the redundant openThreadRegistration() after a successful CPU engine start (only CTimer and PerfEvents implement registerThread(), and both open the gate in start()); make closeThreadRegistration() and waitForHookCalls() private; keep _hook_calls_inflight off _state's cache line; simplify resizeTimers() using the _timers == NULL iff _max_timers == 0 invariant; keep the gate rationale in one comment. - Test that closeThreadRegistration() waits for in-flight hook calls and that the unregistration gate is open while idle; add a bytes() helper. - Shorten the ProfilingEngines.md thread-registration text, drop new line-number references, and recompute the remaining ones. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 25 +++----- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 5 +- ddprof-lib/src/main/cpp/profiler.cpp | 52 ++++++++++----- ddprof-lib/src/main/cpp/profiler.h | 50 ++++++++++----- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 49 +++++++++++++-- doc/architecture/ProfilingEngines.md | 66 ++++++++++---------- 6 files changed, 159 insertions(+), 88 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 112c2af5dc..ec09d907b8 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -132,23 +132,18 @@ void CTimer::unregisterThread(int tid) { } } -// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs from -// start() before it calls Profiler::openThreadRegistration(), so the pthread -// hook's Profiler::registerThread()/unregisterThread() cannot touch the table -// while it is freed and replaced, and JVMTI thread events are not enabled yet. On -// allocation failure the table is left empty (_max_timers == 0), so the tid -// bound checks reject every tid instead of indexing a NULL table, and the next -// start() retries. +// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs before +// start() opens thread registration, so no hook call can be inside the table +// while it is replaced. On allocation failure the table is left empty +// (_max_timers == 0), so the tid bound checks reject every tid instead of +// indexing a NULL table, and the next start() retries. _timers is NULL exactly +// when _max_timers is 0, so the old size is always _max_timers ints. bool CTimer::resizeTimers(int max_timers) { if (max_timers == _max_timers) { return true; } - long long old_bytes = - (_timers != NULL) ? (long long)((size_t)_max_timers * sizeof(int)) : 0; free(_timers); - if (old_bytes > 0) { - NativeMem::record(NM_CTIMER, -old_bytes); - } + NativeMem::record(NM_CTIMER, -(long long)((size_t)_max_timers * sizeof(int))); _timers = (int *)calloc(max_timers, sizeof(int)); if (_timers == NULL) { _max_timers = 0; @@ -194,10 +189,8 @@ Error CTimer::startWithTableSize(Arguments &args, int max_timers) { OS::installSignalHandler(_signal, signalHandler); - // The table is sized, so let the native-thread hook register and unregister - // threads from here on. Opening the gate before listing threads means a - // thread that starts after the list is taken is registered by the hook, and - // one that exits after being registered below is unregistered by it. + // The table is sized: open the hook gate before listing threads (see + // Profiler::openThreadRegistration()). Profiler::instance()->openThreadRegistration(); // Register all existing threads. Individual failures are benign — a thread diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 451a917acc..4de0f227e0 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -980,9 +980,8 @@ Error PerfEvents::start(Arguments &args) { __atomic_store_n(_pthread_entry, (void *)pthread_setspecific_hook, __ATOMIC_RELEASE); - // _events is sized, so let the native-thread hook register and unregister - // threads from here on; opening the gate before listing threads means no - // thread that starts or exits during the loop below is missed. + // _events is sized: open the hook gate before listing threads (see + // Profiler::openThreadRegistration()). Profiler::instance()->openThreadRegistration(); // Create perf_events for all existing threads diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index bed7048b37..14d8f0e441 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -166,11 +166,11 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { } // Marks a native-thread hook call as in flight for the duration of a -// Profiler::registerThread()/unregisterThread() call, so stop() can wait for -// calls that passed the _engines_ready check before it stops the engines. -// The increment and the flag load in the caller are seq_cst, pairing with the -// seq_cst flag store and counter load in stop(): either the hook sees the flag -// cleared, or stop() sees the call in flight and waits for it. +// Profiler::registerThread()/unregisterThread() call, so a gate can be closed +// and the calls that already passed it waited for. The increment and the gate +// load in the caller are seq_cst, pairing with the seq_cst gate store and +// counter load on the closing side: either the hook sees the gate closed, or +// the closer sees the call in flight and waits for it. class HookCallScope { public: explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { @@ -184,13 +184,40 @@ class HookCallScope { std::atomic &_inflight; }; +// Keeps Profiler::unregisterThread() off the engines while start() selects +// the CPU engine and lets it replace its per-thread table. Exits skipped in +// this window leave nothing behind: CTimer::stop() deleted every timer, and +// PerfEvents::start() closes the fd of every tid missing from its fresh +// thread list. The gate is set again by openThreadRegistration() once the +// table is sized, and at the latest when start() returns. +class Profiler::ThreadTablesFreeze { +public: + explicit ThreadTablesFreeze(Profiler *profiler) : _profiler(profiler) { + _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); + _profiler->waitForHookCalls(); + } + ~ThreadTablesFreeze() { + _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); + } + ThreadTablesFreeze(const ThreadTablesFreeze &) = delete; + ThreadTablesFreeze &operator=(const ThreadTablesFreeze &) = delete; + +private: + Profiler *_profiler; +}; + void Profiler::openThreadRegistration() { + _thread_tables_stable.store(true, std::memory_order_seq_cst); _engines_ready.store(true, std::memory_order_seq_cst); } void Profiler::closeThreadRegistration() { - // Hook calls only do a few syscalls per engine, so the wait is short. _engines_ready.store(false, std::memory_order_seq_cst); + waitForHookCalls(); +} + +void Profiler::waitForHookCalls() { + // Hook calls only do a few syscalls per engine, so the wait is short. while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { sched_yield(); } @@ -229,10 +256,10 @@ void Profiler::unregisterThread(int tid) { return; #endif HookCallScope scope(_instance->_hook_calls_inflight); - // Outside the open gate the CPU engine either has not registered any thread - // yet (it opens the gate before listing threads) or stop() is about to - // unregister all of them, so skipping here does not leave a timer behind. - if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { + // Gated only while start() may replace the per-thread tables, not on + // _engines_ready: a thread started during a recording can exit after stop(), + // and PerfEvents keeps its fd open until this call closes it. + if (!_instance->_thread_tables_stable.load(std::memory_order_seq_cst)) { return; } _instance->_cpu_engine->unregisterThread(tid); @@ -1649,6 +1676,7 @@ Error Profiler::start(Arguments &args, bool reset) { _thread_filter.remove(slot_id); // Remove from filtering initially (matches onThreadStart behavior) } + ThreadTablesFreeze tables_freeze(this); _cpu_engine = selectCpuEngine(args); _wall_engine = selectWallEngine(args); _cstack = args._cstack; @@ -1732,10 +1760,6 @@ Error Profiler::start(Arguments &args, bool reset) { error = Error::OK; // recoverable } else { activated |= EM_CPU; - // CTimer and PerfEvents already opened the gate inside start(); this - // covers CPU engines that do not. The wall engines do not implement - // registerThread(), so they do not need to have started yet. - openThreadRegistration(); } } if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 991926f69e..2520758ca2 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -89,14 +89,31 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; - // Set by openThreadRegistration() and cleared by closeThreadRegistration(). registerThread()/unregisterThread() are reached - // from the pthread hook without _state_lock, so they check this flag to - // stay off the engines while start() sizes their per-thread structures (for - // example CTimer::resizeTimers()) and while stop() tears them down. + // Registration gate, set by openThreadRegistration() and cleared by + // closeThreadRegistration(). registerThread() is reached from the pthread + // hook without _state_lock, so it checks this flag to stay off the engines + // while start() sizes their per-thread structures (for example + // CTimer::resizeTimers()) and once stop() has torn them down. std::atomic _engines_ready; - // Hook calls that are past their _engines_ready check and may still be - // inside an engine. closeThreadRegistration() waits for this to reach zero. - std::atomic _hook_calls_inflight; + // Unregistration gate. Cleared by ThreadTablesFreeze for the part of start() + // that may replace the engines' per-thread tables, and set again once they + // are sized. Unlike _engines_ready it stays set while the profiler is idle: + // PerfEvents keeps its per-thread fds across stop(), and only + // unregisterThread() closes the fd of a thread that exits after stop(). + std::atomic _thread_tables_stable; + // Hook calls that are past either gate and may still be inside an engine. + // Closing a gate waits for this to reach zero. Kept off _state's cache line: + // the hook writes it on every native thread start and exit, while + // isRunning() reads _state on every intercepted socket call. + alignas(DEFAULT_CACHE_LINE_SIZE) std::atomic _hook_calls_inflight; + // Clears _thread_tables_stable (waiting for in-flight hook calls) for its + // lifetime; see profiler.cpp. + class ThreadTablesFreeze; + // Closes the gate opened by openThreadRegistration() and waits for hook + // calls already past it to leave the engines. + void closeThreadRegistration(); + // Waits until no hook call is inside an engine. + void waitForHookCalls(); // class unload hook Trap _class_unload_hook_trap; typedef void (*NotifyClassUnloadedFunc)(void *); @@ -258,7 +275,8 @@ class alignas(alignof(SpinLock)) Profiler { public: Profiler() : _state_lock(), _state(State::NEW), _engines_ready(false), - _hook_calls_inflight(0), _class_unload_hook_trap(2), + _thread_tables_stable(true), _hook_calls_inflight(0), + _class_unload_hook_trap(2), _notify_class_unloaded_func(NULL), _thread_info(), _class_map(1), _string_label_map(2), _context_value_map(3), _thread_filter(), _call_trace_storage(), _jfr(), _cpu_engine(NULL), _wall_engine(NULL), @@ -385,11 +403,13 @@ class alignas(alignof(SpinLock)) Profiler { void switchThreadEvents(jvmtiEventMode mode); /** * Lets the native-thread hook's registerThread()/unregisterThread() reach - * the engines. A CPU engine with per-thread structures calls this from its - * start() once those structures are sized and before it lists the existing - * threads, so no thread that starts or exits during start() is missed. - * start() also calls it after a successful CPU engine start, for engines - * that do not call it themselves. Idempotent. + * the engines. A CPU engine with a per-thread table (CTimer, PerfEvents) + * calls this from its start() once the table is sized and before it lists + * the existing threads: a thread that starts after the list is taken is + * then registered by the hook, and one that exits after start() registered + * it is unregistered by the hook. Concurrent registration of one tid by the + * hook and by start() is resolved by the engine's per-slot CAS. Engines + * without registerThread() never need the gate. Idempotent. */ void openThreadRegistration(); @@ -524,10 +544,6 @@ class alignas(alignof(SpinLock)) Profiler { static void setupSignalHandlers(); static void checkFault(ProfiledThread* thrd, siginfo_t *siginfo, void *ucontext); - // Closes the gate opened by openThreadRegistration() and waits for hook - // calls already past it to leave the engines. - void closeThreadRegistration(); - static int registerThread(int tid); static void unregisterThread(int tid); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 1f041b14b4..abaac9d5fe 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -14,7 +14,10 @@ #include "os.h" #include "profiler.h" +#include +#include #include +#include // The failure-path tests force calloc to fail with a negative count, which // makes count * size overflow. The CI sanitizer jobs run gtest binaries @@ -38,6 +41,12 @@ class ProfilerTestAccessor { static void closeThreadRegistration(Profiler *p) { p->closeThreadRegistration(); } + static bool threadTablesStable(Profiler *p) { + return p->_thread_tables_stable.load(std::memory_order_acquire); + } + static void addHookCalls(Profiler *p, int delta) { + p->_hook_calls_inflight.fetch_add(delta, std::memory_order_seq_cst); + } }; // Exposes CTimer's tid-indexed timer table without starting the engine, which @@ -68,6 +77,9 @@ class CTimerTestAccessor { } }; +// Size of a timer table with n slots, as recorded under NM_CTIMER. +static long long bytes(int n) { return (long long)((size_t)n * sizeof(int)); } + class CTimerNativeMemTest : public ::testing::Test { protected: int _saved_max; @@ -95,14 +107,14 @@ class CTimerNativeMemTest : public ::testing::Test { TEST_F(CTimerNativeMemTest, TimerTableIsAccounted) { CTimerTestAccessor::resizeTimers(1024); ASSERT_NE(nullptr, CTimerTestAccessor::timers()); - EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); + EXPECT_EQ(bytes(1024), NativeMem::live(NM_CTIMER)); CTimerTestAccessor::resizeTimers(4096); ASSERT_NE(nullptr, CTimerTestAccessor::timers()); - EXPECT_EQ((long long)(4096 * sizeof(int)), NativeMem::live(NM_CTIMER)); + EXPECT_EQ(bytes(4096), NativeMem::live(NM_CTIMER)); CTimerTestAccessor::resizeTimers(512); - EXPECT_EQ((long long)(512 * sizeof(int)), NativeMem::live(NM_CTIMER)); + EXPECT_EQ(bytes(512), NativeMem::live(NM_CTIMER)); } // Restarting with an unchanged pid_max keeps the table and the gauge as-is. @@ -116,7 +128,7 @@ TEST_F(CTimerNativeMemTest, SameSizeDoesNotReaccount) { ASSERT_TRUE(CTimerTestAccessor::resizeTimers(2048)); EXPECT_EQ(table, CTimerTestAccessor::timers()); EXPECT_EQ(0x5a5a5a5a, CTimerTestAccessor::timers()[2047]); - EXPECT_EQ((long long)(2048 * sizeof(int)), NativeMem::live(NM_CTIMER)); + EXPECT_EQ(bytes(2048), NativeMem::live(NM_CTIMER)); } // A failed allocation leaves an empty table rather than a NULL table with a @@ -134,7 +146,7 @@ TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { ASSERT_TRUE(CTimerTestAccessor::resizeTimers(1024)); EXPECT_NE(nullptr, CTimerTestAccessor::timers()); EXPECT_EQ(1024, CTimerTestAccessor::maxTimers()); - EXPECT_EQ((long long)(1024 * sizeof(int)), NativeMem::live(NM_CTIMER)); + EXPECT_EQ(bytes(1024), NativeMem::live(NM_CTIMER)); } // start() opens the native-thread registration gate once the table is sized, @@ -180,11 +192,38 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { // started, so its engine pointers are null and any engine call would crash. // The call must also leave the in-flight count at zero, or stop() would wait // on it forever. +// +// Unregistration has its own gate, which stays open while the profiler is +// idle: PerfEvents keeps per-thread fds across stop(), and only +// unregisterThread() closes the fd of a thread that exits after stop(). TEST(ProfilerEnginesGateTest, RegisterThreadSkipsEnginesUntilReady) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); EXPECT_EQ(0, Profiler::registerThread(OS::threadId())); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + +// stop() relies on closeThreadRegistration() not returning while a hook call +// that passed the gate may still be inside an engine. +TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { + Profiler *profiler = Profiler::instance(); + profiler->openThreadRegistration(); + ProfilerTestAccessor::addHookCalls(profiler, 1); + + std::atomic closed{false}; + std::thread closer([&] { + ProfilerTestAccessor::closeThreadRegistration(profiler); + closed.store(true); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_FALSE(closed.load()); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + + ProfilerTestAccessor::addHookCalls(profiler, -1); + closer.join(); + EXPECT_TRUE(closed.load()); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index c09d9fb390..9f1cf4bc20 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1314`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1361`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1396`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1341`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1388`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1423`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,13 +65,14 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1498 +Profiler::start() profiler.cpp:1525 + ThreadTablesFreeze (until return) unregisterThread() gate closed selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire _cpu_engine->start() (EM_CPU) failure -> warning, continue CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads - on failure: closeThreadRegistration(); on success: openThreadRegistration() + on failure: closeThreadRegistration() _wall_engine->enableEvents(true) BEFORE start(): see below _wall_engine->start() (EM_WALL) _alloc_engine = selectAllocEngine(); start() (EM_ALLOC) @@ -84,28 +85,28 @@ Profiler::start() profiler.cpp:1498 - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1741-1747`). -- The CPU engine is enabled last (`profiler.cpp:1842`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1765-1771`). +- The CPU engine is enabled last (`profiler.cpp:1866`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1862`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1886`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1049`). + only (`profiler.cpp:1076`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1875`). + and tears nothing down (`profiler.cpp:1899`). 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook - call that passed the check is still inside an engine (`profiler.cpp:1887`). + call that passed the check is still inside an engine. No native thread can then arm a timer after `CTimer::stop()`, or still be using the timer table when the next `start()` resizes it. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2013`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2037`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -122,16 +123,15 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: wraps the start routine (`start_routine_wrapper()`) and calls `Profiler::registerThread()` (`libraryPatcher_linux.cpp:553`). On exit, `unregister_and_release()` calls `Profiler::unregisterThread()` - (`libraryPatcher_linux.cpp:249`). The hook is installed before the engines - start and runs without `_state_lock`, so both calls return without touching - the engines unless `_engines_ready` is set. `CTimer::start()` and - `PerfEvents::start()` set it with `Profiler::openThreadRegistration()` once - their per-thread table is sized and before they list the existing threads - (`ctimer_linux.cpp:201`, `perfEvents_linux.cpp:986`). This keeps the calls - off a table that is still being freed and replaced, and that `stop()` is - tearing down, without missing a thread that starts or exits while `start()` - registers the existing ones. Concurrent registration of one tid by the hook - and by `start()` is resolved by the per-slot CAS. + (`libraryPatcher_linux.cpp:249`). The hook runs without `_state_lock` + and is installed before the engines start, so both calls are gated: + - `registerThread()` needs `_engines_ready`, which `CTimer::start()` and + `PerfEvents::start()` set via `Profiler::openThreadRegistration()` once + their table is sized and before they list threads. `stop()` clears it. + - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only + while `start()` may replace the table (`ThreadTablesFreeze`). It stays + set while idle, because `PerfEvents::stop()` keeps per-thread fds open + and this call closes the fd of a thread that exits after `stop()`. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:980`). @@ -158,7 +158,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:725-736`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:752-763`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -169,7 +169,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1314-1359`): +`selectCpuEngine()` (`profiler.cpp:1341-1386`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -210,19 +210,19 @@ resolved through `Arguments::cpuSamplerInterval()`. with `SIGEV_THREAD_ID` targeting `tid`, carrying `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:183`). +- **Signal**: `SIGPROF` (`ctimer_linux.cpp:178`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:296-300`). + (`ctimer_linux.cpp:289-293`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above `_max_timers` logs a warning and is not profiled. -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:216-222`). +- **`stop()`** unregisters every slot (`ctimer_linux.cpp:209-215`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:231-287`). It has no ASGCT fallback. + (`ctimer_linux.cpp:224-280`). It has no ASGCT fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -243,13 +243,13 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1052`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1051`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1667`, `1687`). + engine (`profiler.cpp:1695`, `1715`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -284,7 +284,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1361-1394`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1388-1421`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -355,7 +355,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:373` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:400` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -424,7 +424,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2091`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2115`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From f79c1cc7037b37ee76edec4248161aff4c7d6fbe Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 16:15:59 -0400 Subject: [PATCH 08/26] PROF-16213: Fail PerfEvents start on table allocation failure; test real hook drain - PerfEvents::start() opened the hook gate even when the _events calloc failed, leaving a NULL table with a non-zero _max_events for the hook's registerThread() and the idle-open unregisterThread() to index. On failure it now returns to the not-started state (_max_events == -1) and returns an error before installing its hook or opening the gate, as CTimer already does. - CloseThreadRegistrationWaitsForHookCalls now holds a real Profiler::registerThread() call inside a blocking test engine instead of bumping the in-flight counter, so removing HookCallScope fails it. - Profiler::registerThread() returns -1 when the gate is closed, matching Engine::registerThread(). - StartOpensThreadRegistration restores the SIGPROF disposition; StartFailsWhenTableAllocationFails checks the handler is unchanged. - Note in resizeTimers() why the JVMTI path cannot race the resize, fix the ThreadTablesFreeze line in the lifecycle diagram, and recompute shifted ctimer_linux.cpp/perfEvents_linux.cpp references. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 6 +- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 13 ++-- ddprof-lib/src/main/cpp/profiler.cpp | 2 +- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 65 +++++++++++++++++--- doc/architecture/ProfilingEngines.md | 16 ++--- 5 files changed, 81 insertions(+), 21 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index ec09d907b8..26ba43dac4 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -133,8 +133,10 @@ void CTimer::unregisterThread(int tid) { } // Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs before -// start() opens thread registration, so no hook call can be inside the table -// while it is replaced. On allocation failure the table is left empty +// start() opens thread registration, so no pthread-hook call can be inside the +// table while it is replaced. The JVMTI ThreadStart/ThreadEnd callbacks reach +// the engine directly, but Profiler enables them only after the engines have +// started and disables them only after the engines have stopped. On allocation failure the table is left empty // (_max_timers == 0), so the tid bound checks reject every tid instead of // indexing a NULL table, and the next start() retries. _timers is NULL exactly // when _max_timers is 0, so the old size is always _max_timers ints. diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 4de0f227e0..290efab04f 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -967,11 +967,16 @@ Error PerfEvents::start(Arguments &args) { NativeMem::record(NM_PERF, -old_bytes); } _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); - _max_events = max_events; - if (_events != NULL) { - NativeMem::record(NM_PERF, - (long long)((size_t)max_events * sizeof(PerfEvent))); + if (_events == NULL) { + // Back to the not-started state (_max_events == -1): registerThread() + // and the tid bound check in unregisterThread() then never index the + // NULL table, and the next start() retries the allocation. + _max_events = -1; + return Error("Failed to allocate perf event table"); } + _max_events = max_events; + NativeMem::record(NM_PERF, + (long long)((size_t)max_events * sizeof(PerfEvent))); } OS::installSignalHandler(SIGPROF, signalHandler); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 14d8f0e441..1a6f6004ed 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -228,7 +228,7 @@ int Profiler::registerThread(int tid) { // Threads created before the gate opens are picked up by the CPU engine's // start(), which opens the gate before it lists the existing threads. if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { - return 0; + return -1; // not registered, as Engine::registerThread() reports it } return _instance->_cpu_engine->registerThread(tid) | _instance->_wall_engine->registerThread(tid); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index abaac9d5fe..3b7a03f099 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -10,12 +10,14 @@ #include "arguments.h" #include "common.h" #include "ctimer.h" +#include "engine.h" #include "nativeMem.h" #include "os.h" #include "profiler.h" #include #include +#include #include #include @@ -44,8 +46,14 @@ class ProfilerTestAccessor { static bool threadTablesStable(Profiler *p) { return p->_thread_tables_stable.load(std::memory_order_acquire); } - static void addHookCalls(Profiler *p, int delta) { - p->_hook_calls_inflight.fetch_add(delta, std::memory_order_seq_cst); + // Installs the given engines and returns the previous ones through the + // out-parameters, so a test can put them back. + static void swapEngines(Profiler *p, Engine *cpu, Engine *wall, + Engine **prev_cpu, Engine **prev_wall) { + *prev_cpu = p->_cpu_engine; + *prev_wall = p->_wall_engine; + p->_cpu_engine = cpu; + p->_wall_engine = wall; } }; @@ -77,6 +85,14 @@ class CTimerTestAccessor { } }; +// Returns the current SIGPROF handler address (sa_handler and sa_sigaction +// share storage), to check whether a call installed a handler. +static void *sigprofHandler() { + struct sigaction sa; + sigaction(SIGPROF, nullptr, &sa); + return (void *)sa.sa_sigaction; +} + // Size of a timer table with n slots, as recorded under NM_CTIMER. static long long bytes(int n) { return (long long)((size_t)n * sizeof(int)); } @@ -152,10 +168,13 @@ TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { // start() opens the native-thread registration gate once the table is sized, // so a thread that starts or exits while start() registers the existing // threads is handled by the pthread hook instead of being missed. The engine -// is stopped and the gate closed again before the fixture restores the table. +// is stopped, the gate closed and the SIGPROF disposition restored before the +// fixture restores the table, so later tests see the process as it was. TEST_F(CTimerNativeMemTest, StartOpensThreadRegistration) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + struct sigaction saved_sigprof; + sigaction(SIGPROF, nullptr, &saved_sigprof); CTimer engine; Arguments args; @@ -164,6 +183,7 @@ TEST_F(CTimerNativeMemTest, StartOpensThreadRegistration) { bool opened = ProfilerTestAccessor::enginesReady(profiler); engine.stop(); ProfilerTestAccessor::closeThreadRegistration(profiler); + sigaction(SIGPROF, &saved_sigprof, nullptr); EXPECT_FALSE((bool)error); EXPECT_TRUE(opened); @@ -175,9 +195,11 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { SKIP_IF_CALLOC_OVERFLOW_ABORTS(); CTimer engine; Arguments args; + void *handler_before = sigprofHandler(); Error error = CTimerTestAccessor::startWithTableSize(engine, args, -1); EXPECT_TRUE((bool)error); + EXPECT_EQ(handler_before, sigprofHandler()); EXPECT_FALSE(ProfilerTestAccessor::enginesReady(Profiler::instance())); EXPECT_STREQ("Failed to allocate CPU timer table", error.message()); EXPECT_EQ(nullptr, CTimerTestAccessor::timers()); @@ -199,18 +221,43 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { TEST(ProfilerEnginesGateTest, RegisterThreadSkipsEnginesUntilReady) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); - EXPECT_EQ(0, Profiler::registerThread(OS::threadId())); + EXPECT_EQ(-1, Profiler::registerThread(OS::threadId())); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); } +// A CPU engine whose registerThread() blocks until released, to hold a real +// Profiler::registerThread() call inside the engine. +class BlockingEngine : public Engine { +public: + std::atomic entered{false}; + std::atomic released{false}; + + int registerThread(int tid) override { + entered.store(true); + while (!released.load()) { + std::this_thread::yield(); + } + return 0; + } +}; + // stop() relies on closeThreadRegistration() not returning while a hook call -// that passed the gate may still be inside an engine. +// that passed the gate is still inside an engine; otherwise CTimer::stop() or +// the next resizeTimers() could run under it. TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { Profiler *profiler = Profiler::instance(); + BlockingEngine cpu; + Engine wall; + Engine *prev_cpu, *prev_wall; + ProfilerTestAccessor::swapEngines(profiler, &cpu, &wall, &prev_cpu, + &prev_wall); profiler->openThreadRegistration(); - ProfilerTestAccessor::addHookCalls(profiler, 1); + std::thread hook([] { Profiler::registerThread(OS::threadId()); }); + while (!cpu.entered.load()) { + std::this_thread::yield(); + } std::atomic closed{false}; std::thread closer([&] { ProfilerTestAccessor::closeThreadRegistration(profiler); @@ -220,10 +267,14 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { EXPECT_FALSE(closed.load()); EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); - ProfilerTestAccessor::addHookCalls(profiler, -1); + cpu.released.store(true); + hook.join(); closer.join(); EXPECT_TRUE(closed.load()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + Engine *ignored_cpu, *ignored_wall; + ProfilerTestAccessor::swapEngines(profiler, prev_cpu, prev_wall, + &ignored_cpu, &ignored_wall); } #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 9f1cf4bc20..4a70c69077 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -66,7 +66,9 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ``` Profiler::start() profiler.cpp:1525 - ThreadTablesFreeze (until return) unregisterThread() gate closed + ThreadTablesFreeze unregisterThread() gate closed until + the CPU engine sizes its table, at the + latest until return selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire @@ -134,7 +136,7 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: and this call closes the fd of a thread that exits after `stop()`. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` - (`perfEvents_linux.cpp:980`). + (`perfEvents_linux.cpp:985`). Each engine's `start()` also registers every thread that already exists. @@ -210,19 +212,19 @@ resolved through `Arguments::cpuSamplerInterval()`. with `SIGEV_THREAD_ID` targeting `tid`, carrying `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:178`). +- **Signal**: `SIGPROF` (`ctimer_linux.cpp:180`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:289-293`). + (`ctimer_linux.cpp:291-295`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above `_max_timers` logs a warning and is not profiled. -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:209-215`). +- **`stop()`** unregisters every slot (`ctimer_linux.cpp:211-217`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:224-280`). It has no ASGCT fallback. + (`ctimer_linux.cpp:226-282`). It has no ASGCT fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -243,7 +245,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1051`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1056`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted From eb6b11f28df4dfa3dd2d86ffcf30a850176979ca Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 17:24:30 -0400 Subject: [PATCH 09/26] PROF-16213: Disable cancellation in hook calls; guard resetBuffer; test unregister gate - HookCallScope disables thread cancellation for the hook call. The engine calls reach cancellation points (close(), Log::warn()); acting on a cancellation there would abort while unwinding through the destructor with the statically linked libgcc, or skip it on musl and leave the in-flight count raised so the next stop() waits forever. - PerfEvents::resetBuffer(), called from the SIGPROF handler for fds left armed by an earlier recording, checks the table and the bound before indexing, so a failed _events allocation in start() cannot be indexed. - Move the UNIT_TEST early return in Profiler::unregisterThread() below the gate and add UnregisterThreadSkipsEnginesWhileTablesFrozen. - Say in the stop() comment, the openThreadRegistration() doc and ProfilingEngines.md that JVMTI ThreadStart can still arm a timer after CTimer::stop(), and that PerfEvents::start() closes the fd of a thread the hook registered after the thread list was taken. Reflow the resizeTimers() comment and drop its NULL-iff-zero claim. - Tests: ignore SIGPROF before CTimer::stop(), bound the blocking-engine waits with a deadline, restore engine pointers with an RAII guard. - Recompute shifted line references in ProfilingEngines.md. The remaining PerfEvents issues are tracked in PROF-16359. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 10 ++- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 11 ++- ddprof-lib/src/main/cpp/profiler.cpp | 44 ++++++---- ddprof-lib/src/main/cpp/profiler.h | 12 +-- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 84 +++++++++++++++----- doc/architecture/ProfilingEngines.md | 48 ++++++----- 6 files changed, 143 insertions(+), 66 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 26ba43dac4..9c2d509034 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -136,10 +136,12 @@ void CTimer::unregisterThread(int tid) { // start() opens thread registration, so no pthread-hook call can be inside the // table while it is replaced. The JVMTI ThreadStart/ThreadEnd callbacks reach // the engine directly, but Profiler enables them only after the engines have -// started and disables them only after the engines have stopped. On allocation failure the table is left empty -// (_max_timers == 0), so the tid bound checks reject every tid instead of -// indexing a NULL table, and the next start() retries. _timers is NULL exactly -// when _max_timers is 0, so the old size is always _max_timers ints. +// started and disables them only after the engines have stopped. +// +// On allocation failure the table is left empty (_max_timers == 0), so the tid +// bound checks reject every tid instead of indexing a NULL table, and the next +// start() retries. The table currently held always has _max_timers slots, so +// that is the size released from NM_CTIMER. bool CTimer::resizeTimers(int max_timers) { if (max_timers == _max_timers) { return true; diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 290efab04f..0689f43c87 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -1157,7 +1157,16 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, } void PerfEvents::resetBuffer(int tid) { - PerfEvent *event = &_events[tid]; + // Runs from the SIGPROF handler on every exit, including signals from fds + // left armed by an earlier recording. start() can leave the table NULL + // (_max_events == -1) after a failed allocation, so check the table and the + // bound before indexing it. + PerfEvent *events = __atomic_load_n(&_events, __ATOMIC_ACQUIRE); + if (events == NULL || tid < 0 || + tid >= __atomic_load_n(&_max_events, __ATOMIC_ACQUIRE)) { + return; + } + PerfEvent *event = &events[tid]; if (!event->tryLock()) { return; // the event is being destroyed } diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 1a6f6004ed..6f08dcb1e7 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -171,17 +172,29 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // load in the caller are seq_cst, pairing with the seq_cst gate store and // counter load on the closing side: either the hook sees the gate closed, or // the closer sees the call in flight and waits for it. +// +// Thread cancellation is disabled for the scope's lifetime: the engine calls +// reach cancellation points (close() in PerfEvents::unregisterThread(), stdio +// in Log::warn()). Acting on a cancellation there would unwind through this +// destructor, which aborts with the statically linked libgcc (see +// run_with_cleanup() in libraryPatcher_linux.cpp), or skip it on musl and +// leave the count raised, so the next stop() would wait forever. class HookCallScope { public: explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &_cancel_state); _inflight.fetch_add(1, std::memory_order_seq_cst); } - ~HookCallScope() { _inflight.fetch_sub(1, std::memory_order_release); } + ~HookCallScope() { + _inflight.fetch_sub(1, std::memory_order_release); + pthread_setcancelstate(_cancel_state, nullptr); + } HookCallScope(const HookCallScope &) = delete; HookCallScope &operator=(const HookCallScope &) = delete; private: std::atomic &_inflight; + int _cancel_state; }; // Keeps Profiler::unregisterThread() off the engines while start() selects @@ -245,16 +258,6 @@ void Profiler::resetUnregisterObservableForTest() { #endif void Profiler::unregisterThread(int tid) { -#ifdef UNIT_TEST - // In gtest, _cpu_engine/_wall_engine are null (profiler not started). - // Record the tid so integration tests can verify the call happened without - // crashing on the null engine dereference. This bypasses the real engine - // unregister path entirely, so that path is covered only by JVM-level tests, - // not by these gtests. UNIT_TEST is defined solely for the gtest binaries - // (see GtestTaskBuilder); the shipped library never compiles this branch. - g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); - return; -#endif HookCallScope scope(_instance->_hook_calls_inflight); // Gated only while start() may replace the per-thread tables, not on // _engines_ready: a thread started during a recording can exit after stop(), @@ -262,6 +265,16 @@ void Profiler::unregisterThread(int tid) { if (!_instance->_thread_tables_stable.load(std::memory_order_seq_cst)) { return; } +#ifdef UNIT_TEST + // In gtest, _cpu_engine/_wall_engine are null (profiler not started). + // Record the tid so integration tests can verify the call passed the gate + // without crashing on the null engine dereference. This bypasses the real + // engine unregister path, so that path is covered only by JVM-level tests, + // not by these gtests. UNIT_TEST is defined solely for the gtest binaries + // (see GtestTaskBuilder); the shipped library never compiles this branch. + g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); + return; +#endif _instance->_cpu_engine->unregisterThread(tid); _instance->_wall_engine->unregisterThread(tid); } @@ -1905,9 +1918,12 @@ Error Profiler::stop() { return Error("signal handlers did not drain; teardown skipped, retry stop()"); } - // Close the hook gate and wait for hook calls already past it, so no thread - // arms a timer after CTimer::stop() has unregistered them all, and none is - // still inside the timer table when the next start() resizes it. + // Close the hook gate and wait for hook calls already past it, so no + // pthread-hook call arms a timer after CTimer::stop() has unregistered them + // all, and none is still inside the timer table when the next start() + // resizes it. JVMTI ThreadStart reaches the engine directly and is disabled + // only after the engines stop, so a Java thread starting in that window can + // still arm a timer that nothing deletes. closeThreadRegistration(); if (_event_mask & EM_ALLOC) diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 2520758ca2..ed153d58e9 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -405,11 +405,13 @@ class alignas(alignof(SpinLock)) Profiler { * Lets the native-thread hook's registerThread()/unregisterThread() reach * the engines. A CPU engine with a per-thread table (CTimer, PerfEvents) * calls this from its start() once the table is sized and before it lists - * the existing threads: a thread that starts after the list is taken is - * then registered by the hook, and one that exits after start() registered - * it is unregistered by the hook. Concurrent registration of one tid by the - * hook and by start() is resolved by the engine's per-slot CAS. Engines - * without registerThread() never need the gate. Idempotent. + * the existing threads, so the hook handles a thread that starts or exits + * while start() registers the existing ones. For CTimer this misses no + * thread: the per-slot CAS resolves the hook and start() registering one + * tid. PerfEvents::start() still closes the fd of every tid missing from + * its thread list, including one the hook registered after the list was + * taken, so such a thread is not sampled. Engines without registerThread() + * never need the gate. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 3b7a03f099..41645bd0f4 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -46,17 +46,46 @@ class ProfilerTestAccessor { static bool threadTablesStable(Profiler *p) { return p->_thread_tables_stable.load(std::memory_order_acquire); } - // Installs the given engines and returns the previous ones through the - // out-parameters, so a test can put them back. - static void swapEngines(Profiler *p, Engine *cpu, Engine *wall, - Engine **prev_cpu, Engine **prev_wall) { - *prev_cpu = p->_cpu_engine; - *prev_wall = p->_wall_engine; - p->_cpu_engine = cpu; - p->_wall_engine = wall; + static void setThreadTablesStable(Profiler *p, bool stable) { + p->_thread_tables_stable.store(stable, std::memory_order_seq_cst); } + + // Installs the given engines for its lifetime and puts the previous ones + // back on destruction, including when a test assertion returns early. + class EngineSwap { + public: + EngineSwap(Profiler *p, Engine *cpu, Engine *wall) + : _p(p), _cpu(p->_cpu_engine), _wall(p->_wall_engine) { + p->_cpu_engine = cpu; + p->_wall_engine = wall; + } + ~EngineSwap() { + _p->_cpu_engine = _cpu; + _p->_wall_engine = _wall; + } + EngineSwap(const EngineSwap &) = delete; + EngineSwap &operator=(const EngineSwap &) = delete; + + private: + Profiler *_p; + Engine *_cpu; + Engine *_wall; + }; }; +// Polls cond until it holds or a 5 s deadline passes, so a gate regression +// fails the test instead of hanging the gtest binary. +template static bool waitFor(Cond cond) { + auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (!cond()) { + if (std::chrono::steady_clock::now() > deadline) { + return false; + } + std::this_thread::yield(); + } + return true; +} + // Exposes CTimer's tid-indexed timer table without starting the engine, which // would install a SIGPROF handler and arm a timer for every live thread. class CTimerTestAccessor { @@ -181,6 +210,9 @@ TEST_F(CTimerNativeMemTest, StartOpensThreadRegistration) { Error error = CTimerTestAccessor::startWithTableSize(engine, args, OS::getMaxThreadId()); bool opened = ProfilerTestAccessor::enginesReady(profiler); + // Ignore SIGPROF before stopping, so a signal an armed timer already + // generated is discarded instead of hitting a restored SIG_DFL. + signal(SIGPROF, SIG_IGN); engine.stop(); ProfilerTestAccessor::closeThreadRegistration(profiler); sigaction(SIGPROF, &saved_sigprof, nullptr); @@ -235,9 +267,7 @@ class BlockingEngine : public Engine { int registerThread(int tid) override { entered.store(true); - while (!released.load()) { - std::this_thread::yield(); - } + waitFor([this] { return released.load(); }); return 0; } }; @@ -249,15 +279,11 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { Profiler *profiler = Profiler::instance(); BlockingEngine cpu; Engine wall; - Engine *prev_cpu, *prev_wall; - ProfilerTestAccessor::swapEngines(profiler, &cpu, &wall, &prev_cpu, - &prev_wall); + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); profiler->openThreadRegistration(); std::thread hook([] { Profiler::registerThread(OS::threadId()); }); - while (!cpu.entered.load()) { - std::this_thread::yield(); - } + bool entered = waitFor([&] { return cpu.entered.load(); }); std::atomic closed{false}; std::thread closer([&] { ProfilerTestAccessor::closeThreadRegistration(profiler); @@ -271,10 +297,28 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { hook.join(); closer.join(); EXPECT_TRUE(closed.load()); + EXPECT_TRUE(entered); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// Unregistration is skipped, without touching the engines, while start() may +// replace their per-thread tables, and goes through once the tables are +// stable. In this binary the call records the tid instead of calling the +// (null) engines, so the recorded tid shows whether it passed the gate. +TEST(ProfilerEnginesGateTest, UnregisterThreadSkipsEnginesWhileTablesFrozen) { + Profiler *profiler = Profiler::instance(); + int tid = OS::threadId(); + + ProfilerTestAccessor::setThreadTablesStable(profiler, false); + Profiler::resetUnregisterObservableForTest(); + Profiler::unregisterThread(tid); + EXPECT_EQ(-1, Profiler::lastUnregisteredTidForTest()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); - Engine *ignored_cpu, *ignored_wall; - ProfilerTestAccessor::swapEngines(profiler, prev_cpu, prev_wall, - &ignored_cpu, &ignored_wall); + + ProfilerTestAccessor::setThreadTablesStable(profiler, true); + Profiler::unregisterThread(tid); + EXPECT_EQ(tid, Profiler::lastUnregisteredTidForTest()); + Profiler::resetUnregisterObservableForTest(); } #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 4a70c69077..0bce1e95dc 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1341`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1388`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1423`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1354`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1401`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1436`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,7 +65,7 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1525 +Profiler::start() profiler.cpp:1538 ThreadTablesFreeze unregisterThread() gate closed until the CPU engine sizes its table, at the latest until return @@ -87,28 +87,30 @@ Profiler::start() profiler.cpp:1525 - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1765-1771`). -- The CPU engine is enabled last (`profiler.cpp:1866`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1778-1784`). +- The CPU engine is enabled last (`profiler.cpp:1879`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1886`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1899`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1076`). + only (`profiler.cpp:1089`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1899`). + and tears nothing down (`profiler.cpp:1912`). 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook call that passed the check is still inside an engine. - No native thread can then arm a timer after `CTimer::stop()`, or still be - using the timer table when the next `start()` resizes it. + No pthread-hook call can then arm a timer after `CTimer::stop()`, or still + be using the timer table when the next `start()` resizes it. JVMTI + `ThreadStart` is disabled only after the engines stop, so a Java thread + starting in that window can still arm a timer that nothing deletes. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2037`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2053`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -130,6 +132,8 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: - `registerThread()` needs `_engines_ready`, which `CTimer::start()` and `PerfEvents::start()` set via `Profiler::openThreadRegistration()` once their table is sized and before they list threads. `stop()` clears it. + `PerfEvents::start()` still closes the fd of every tid missing from its + thread list, including one the hook registered after the list was taken. - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only while `start()` may replace the table (`ThreadTablesFreeze`). It stays set while idle, because `PerfEvents::stop()` keeps per-thread fds open @@ -160,7 +164,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:752-763`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:765-776`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -171,7 +175,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1341-1386`): +`selectCpuEngine()` (`profiler.cpp:1354-1399`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -212,19 +216,19 @@ resolved through `Arguments::cpuSamplerInterval()`. with `SIGEV_THREAD_ID` targeting `tid`, carrying `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:180`). +- **Signal**: `SIGPROF` (`ctimer_linux.cpp:182`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:291-295`). + (`ctimer_linux.cpp:293-297`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above `_max_timers` logs a warning and is not profiled. -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:211-217`). +- **`stop()`** unregisters every slot (`ctimer_linux.cpp:213-219`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:226-282`). It has no ASGCT fallback. + (`ctimer_linux.cpp:228-284`). It has no ASGCT fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -251,7 +255,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1695`, `1715`). + engine (`profiler.cpp:1708`, `1728`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -286,7 +290,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1388-1421`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1401-1434`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -357,7 +361,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:400` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:413` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -426,7 +430,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2115`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2131`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From ac310d7c5aaa171421ad81f5878afb50226d2ee8 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 19:32:08 -0400 Subject: [PATCH 10/26] PROF-16213: Restore thread cancellation outside destructor-bearing scopes HookCallScope restored the cancellation state from its destructor. With asynchronous cancellation, current glibc acts on a pending request as soon as pthread_setcancelstate() re-enables it, so a cancellation requested during a hook call unwound out of the implicitly noexcept destructor and terminated the process. - Keep HookCallScope for the in-flight count only. Profiler:: registerThread()/unregisterThread() disable cancellation, call new registerThreadInEngines()/unregisterThreadInEngines() helpers that hold every destructor-bearing scope, and restore it only after they return. The manual pair is documented as required by the RAII rule. - Add CancellationDuringHookCallActsAfterTheCall: a thread with asynchronous cancellation is cancelled inside a hook call; the process must survive, the thread must not run past its next cancellation point, and the in-flight count must return to zero. With the restore back in the destructor the process aborts on glibc 2.39; glibc 2.35 keeps the request pending until the next cancellation point, so it cannot catch that regression. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/profiler.cpp | 44 +++++++++++++++-------- ddprof-lib/src/main/cpp/profiler.h | 4 +++ ddprof-lib/src/test/cpp/ctimer_ut.cpp | 50 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 14 deletions(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 6f08dcb1e7..a51cbfa857 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -172,29 +172,17 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // load in the caller are seq_cst, pairing with the seq_cst gate store and // counter load on the closing side: either the hook sees the gate closed, or // the closer sees the call in flight and waits for it. -// -// Thread cancellation is disabled for the scope's lifetime: the engine calls -// reach cancellation points (close() in PerfEvents::unregisterThread(), stdio -// in Log::warn()). Acting on a cancellation there would unwind through this -// destructor, which aborts with the statically linked libgcc (see -// run_with_cleanup() in libraryPatcher_linux.cpp), or skip it on musl and -// leave the count raised, so the next stop() would wait forever. class HookCallScope { public: explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { - pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &_cancel_state); _inflight.fetch_add(1, std::memory_order_seq_cst); } - ~HookCallScope() { - _inflight.fetch_sub(1, std::memory_order_release); - pthread_setcancelstate(_cancel_state, nullptr); - } + ~HookCallScope() { _inflight.fetch_sub(1, std::memory_order_release); } HookCallScope(const HookCallScope &) = delete; HookCallScope &operator=(const HookCallScope &) = delete; private: std::atomic &_inflight; - int _cancel_state; }; // Keeps Profiler::unregisterThread() off the engines while start() selects @@ -236,7 +224,35 @@ void Profiler::waitForHookCalls() { } } +// Thread cancellation is disabled for the whole hook call. The engine calls +// reach cancellation points (close() in PerfEvents::unregisterThread(), stdio +// in Log::warn()); acting on a cancellation there would unwind through +// ~HookCallScope(), which aborts with the statically linked libgcc (see +// run_with_cleanup() in libraryPatcher_linux.cpp), or skip it on musl and +// leave the in-flight count raised, so the next stop() would wait forever. +// +// This is a manual disable/restore pair rather than a guard on purpose: with +// asynchronous cancellation, current glibc's pthread_setcancelstate() acts on a +// pending request as soon as cancellation is re-enabled, so the restore must +// not run inside a destructor, which is implicitly noexcept and would +// terminate. It runs here, after every destructor-bearing scope of the call +// has ended. int Profiler::registerThread(int tid) { + int cancel_state; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state); + int result = registerThreadInEngines(tid); + pthread_setcancelstate(cancel_state, nullptr); + return result; +} + +void Profiler::unregisterThread(int tid) { + int cancel_state; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &cancel_state); + unregisterThreadInEngines(tid); + pthread_setcancelstate(cancel_state, nullptr); +} + +int Profiler::registerThreadInEngines(int tid) { HookCallScope scope(_instance->_hook_calls_inflight); // Threads created before the gate opens are picked up by the CPU engine's // start(), which opens the gate before it lists the existing threads. @@ -257,7 +273,7 @@ void Profiler::resetUnregisterObservableForTest() { } #endif -void Profiler::unregisterThread(int tid) { +void Profiler::unregisterThreadInEngines(int tid) { HookCallScope scope(_instance->_hook_calls_inflight); // Gated only while start() may replace the per-thread tables, not on // _engines_ready: a thread started during a recording can exit after stop(), diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index ed153d58e9..0f941fe41d 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -114,6 +114,10 @@ class alignas(alignof(SpinLock)) Profiler { void closeThreadRegistration(); // Waits until no hook call is inside an engine. void waitForHookCalls(); + // Gate checks and engine calls behind registerThread()/unregisterThread(), + // run with thread cancellation disabled. + static int registerThreadInEngines(int tid); + static void unregisterThreadInEngines(int tid); // class unload hook Trap _class_unload_hook_trap; typedef void (*NotifyClassUnloadedFunc)(void *); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 41645bd0f4..6c3c56c7d5 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include // The failure-path tests force calloc to fail with a negative count, which @@ -301,6 +302,55 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } +// Thread body for the cancellation test: makes a hook call with asynchronous +// cancellation enabled, then reaches a cancellation point. A plain C-style +// thread is used so that no C++ frame with cleanups sits above the call when +// the cancellation unwinds it. +static std::atomic survived_cancellation{false}; +static void *registerWithAsyncCancel(void *) { + pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, nullptr); + Profiler::registerThread(OS::threadId()); + pthread_testcancel(); + survived_cancellation.store(true); + return nullptr; +} + +// A cancellation requested during a hook call stays pending until the call has +// left every destructor-bearing scope. Current glibc acts on a pending +// asynchronous request as soon as cancellation is re-enabled, so re-enabling it +// from a destructor would unwind out of an implicitly noexcept function and +// terminate the process. Older glibc (2.35) keeps it pending until the next +// cancellation point instead. Either way the process must survive, the thread +// must not run past that cancellation point, and the in-flight count must be +// back at zero. The join value is not checked: a forced unwind that crosses +// differently versioned libgcc copies (see run_with_cleanup() in +// libraryPatcher_linux.cpp) does not always report PTHREAD_CANCELED. +TEST(ProfilerEnginesGateTest, CancellationDuringHookCallActsAfterTheCall) { +#if defined(TSAN_ENABLED) + GTEST_SKIP() << "TSan does not support asynchronous thread cancellation"; +#endif + Profiler *profiler = Profiler::instance(); + BlockingEngine cpu; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + profiler->openThreadRegistration(); + + survived_cancellation.store(false); + pthread_t thread; + ASSERT_EQ(0, pthread_create(&thread, nullptr, registerWithAsyncCancel, + nullptr)); + bool entered = waitFor([&] { return cpu.entered.load(); }); + // Cancellation is disabled inside the hook call, so this stays pending. + EXPECT_EQ(0, pthread_cancel(thread)); + cpu.released.store(true); + EXPECT_EQ(0, pthread_join(thread, nullptr)); + ProfilerTestAccessor::closeThreadRegistration(profiler); + + EXPECT_TRUE(entered); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + EXPECT_FALSE(survived_cancellation.load()); +} + // Unregistration is skipped, without touching the engines, while start() may // replace their per-thread tables, and goes through once the tables are // stable. In this binary the call records the tid instead of calling the From ae8c8edd2dc868c3389e9dac8e220392ba8f0927 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Tue, 27 Oct 2026 20:27:29 -0400 Subject: [PATCH 11/26] PROF-16213: Gate JVMTI thread callbacks; prove cancellation disable; fix doc refs - Profiler::onThreadStart()/onThreadEnd() call Profiler::registerThread()/ unregisterThread() instead of the engines directly, so the JVMTI path shares the gates, the in-flight count and the cancellation handling. Disabling the JVMTI events in stop() does not wait for callbacks already running; with the gate, a Java thread starting during stop() can no longer arm a timer after CTimer::stop(), and a failed CPU engine start no longer logs "tid > pid_max[0]" for every Java thread. - The cancellation test now asserts the engine runs with cancellation disabled; the hook-drain test waits until the gate is closed before its timed check. - Correct the ThreadTablesFreeze comment (an exiting thread still listed by the engine leaves its slot behind), the PerfEvents allocation-failure comment, and the CTimerTestAccessor comment; reference PROF-16359 for the known PerfEvents gaps. - Recompute profiler.cpp/ctimer_linux.cpp/perfEvents_linux.cpp line references in ProfilingEngines.md, and the profiler.cpp/profiler.h references in JfrIntegration.md and JvmSupportLayer.md that this branch had shifted. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 7 +-- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 8 ++- ddprof-lib/src/main/cpp/profiler.cpp | 34 ++++++------ ddprof-lib/src/main/cpp/profiler.h | 4 +- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 21 ++++++- doc/architecture/JfrIntegration.md | 16 +++--- doc/architecture/JvmSupportLayer.md | 14 ++--- doc/architecture/ProfilingEngines.md | 58 ++++++++++---------- 8 files changed, 89 insertions(+), 73 deletions(-) diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 9c2d509034..ee133ebcaf 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -133,10 +133,9 @@ void CTimer::unregisterThread(int tid) { } // Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs before -// start() opens thread registration, so no pthread-hook call can be inside the -// table while it is replaced. The JVMTI ThreadStart/ThreadEnd callbacks reach -// the engine directly, but Profiler enables them only after the engines have -// started and disables them only after the engines have stopped. +// start() opens thread registration, so no thread-hook call (pthread hook or +// JVMTI ThreadStart/ThreadEnd, which share the gate) can be inside the table +// while it is replaced. // // On allocation failure the table is left empty (_max_timers == 0), so the tid // bound checks reject every tid instead of indexing a NULL table, and the next diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 0689f43c87..894b4034b2 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -968,9 +968,11 @@ Error PerfEvents::start(Arguments &args) { } _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); if (_events == NULL) { - // Back to the not-started state (_max_events == -1): registerThread() - // and the tid bound check in unregisterThread() then never index the - // NULL table, and the next start() retries the allocation. + // _max_events == -1 makes registerThread() treat the engine as not + // started, and the tid bound checks in unregisterThread() and + // resetBuffer() reject every tid, so nothing indexes the NULL table; the + // next start() retries the allocation. Fds left open by an earlier + // recording are not closed here (PROF-16359). _max_events = -1; return Error("Failed to allocate perf event table"); } diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index a51cbfa857..ffee2fbac4 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -110,8 +110,9 @@ void Profiler::onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { } } - _cpu_engine->registerThread(tid); - _wall_engine->registerThread(tid); + // Through the same gate and in-flight count as the pthread hook: disabling + // the JVMTI event in stop() does not wait for callbacks already running. + registerThread(tid); } void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { @@ -142,8 +143,7 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // partially-torn-down thread (PROF-14674). { blockProfilingForExit(); - _cpu_engine->unregisterThread(tid); - _wall_engine->unregisterThread(tid); + unregisterThread(tid); LivenessTracker::instance()->releaseThreadLocalState(); ProfiledThread::release(); } @@ -161,8 +161,7 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { if (_reference_chains_active) { ReferenceChainTracker::instance()->unregisterThreadObject(jni, tid); } - _cpu_engine->unregisterThread(tid); - _wall_engine->unregisterThread(tid); + unregisterThread(tid); LivenessTracker::instance()->releaseThreadLocalState(); } @@ -186,11 +185,13 @@ class HookCallScope { }; // Keeps Profiler::unregisterThread() off the engines while start() selects -// the CPU engine and lets it replace its per-thread table. Exits skipped in -// this window leave nothing behind: CTimer::stop() deleted every timer, and -// PerfEvents::start() closes the fd of every tid missing from its fresh -// thread list. The gate is set again by openThreadRegistration() once the -// table is sized, and at the latest when start() returns. +// the CPU engine and lets it replace its per-thread table. The gate is set +// again by openThreadRegistration() once the table is sized, and at the latest +// when start() returns. An exit skipped in this window is cleaned up only if +// the thread is gone when the engine lists threads: CTimer::stop() deleted +// every timer, and PerfEvents::start() closes the fd of every listed-missing +// tid. A thread that is still listed while it exits is registered by start() +// and its slot stays occupied after it dies (PROF-16359). class Profiler::ThreadTablesFreeze { public: explicit ThreadTablesFreeze(Profiler *profiler) : _profiler(profiler) { @@ -1934,12 +1935,11 @@ Error Profiler::stop() { return Error("signal handlers did not drain; teardown skipped, retry stop()"); } - // Close the hook gate and wait for hook calls already past it, so no - // pthread-hook call arms a timer after CTimer::stop() has unregistered them - // all, and none is still inside the timer table when the next start() - // resizes it. JVMTI ThreadStart reaches the engine directly and is disabled - // only after the engines stop, so a Java thread starting in that window can - // still arm a timer that nothing deletes. + // Close the registration gate and wait for calls already past it, so no + // thread arms a timer after CTimer::stop() has unregistered them all, and + // none is still inside the timer table when the next start() resizes it. + // This covers the JVMTI ThreadStart/ThreadEnd callbacks too: they go through + // the same gate, and their events are disabled only after the engines stop. closeThreadRegistration(); if (_event_mask & EM_ALLOC) diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 0f941fe41d..509603a1df 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -414,8 +414,8 @@ class alignas(alignof(SpinLock)) Profiler { * thread: the per-slot CAS resolves the hook and start() registering one * tid. PerfEvents::start() still closes the fd of every tid missing from * its thread list, including one the hook registered after the list was - * taken, so such a thread is not sampled. Engines without registerThread() - * never need the gate. Idempotent. + * taken, so such a thread is not sampled (PROF-16359). Engines without + * registerThread() never need the gate. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 6c3c56c7d5..ab7abfc941 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -87,8 +87,10 @@ template static bool waitFor(Cond cond) { return true; } -// Exposes CTimer's tid-indexed timer table without starting the engine, which -// would install a SIGPROF handler and arm a timer for every live thread. +// Exposes CTimer's tid-indexed timer table. The table accessors work without +// starting the engine; startWithTableSize() does start it (installs the +// SIGPROF handler and arms a timer for every live thread), and only +// StartOpensThreadRegistration uses it on a real table. class CTimerTestAccessor { public: static bool resizeTimers(int max_timers) { return CTimer::resizeTimers(max_timers); } @@ -265,8 +267,14 @@ class BlockingEngine : public Engine { public: std::atomic entered{false}; std::atomic released{false}; + // Thread cancellation state seen inside the call (PTHREAD_CANCEL_*), or -1. + std::atomic cancel_state_inside{-1}; int registerThread(int tid) override { + int state; + pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &state); + pthread_setcancelstate(state, nullptr); + cancel_state_inside.store(state); entered.store(true); waitFor([this] { return released.load(); }); return 0; @@ -290,9 +298,12 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { ProfilerTestAccessor::closeThreadRegistration(profiler); closed.store(true); }); + // Wait until the closer has cleared the gate, so it is in the wait loop. + bool gate_closed = + waitFor([&] { return !ProfilerTestAccessor::enginesReady(profiler); }); std::this_thread::sleep_for(std::chrono::milliseconds(50)); + EXPECT_TRUE(gate_closed); EXPECT_FALSE(closed.load()); - EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); cpu.released.store(true); hook.join(); @@ -347,6 +358,10 @@ TEST(ProfilerEnginesGateTest, CancellationDuringHookCallActsAfterTheCall) { ProfilerTestAccessor::closeThreadRegistration(profiler); EXPECT_TRUE(entered); + // The hook call ran with cancellation disabled; without that, the request + // could act at a cancellation point inside the engine and unwind through + // ~HookCallScope(). + EXPECT_EQ(PTHREAD_CANCEL_DISABLE, cpu.cancel_state_inside.load()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); EXPECT_FALSE(survived_cancellation.load()); } diff --git a/doc/architecture/JfrIntegration.md b/doc/architecture/JfrIntegration.md index aec55b2abb..240be4778a 100644 --- a/doc/architecture/JfrIntegration.md +++ b/doc/architecture/JfrIntegration.md @@ -63,9 +63,9 @@ event silently. not by thread: 1. `Profiler::getLockIndex(tid)` hashes the thread ID to a stripe - (`profiler.cpp:229-234`). + (`profiler.cpp:331-336`). 2. The caller `tryLock()`s that stripe, then up to two others; if all three are - taken the sample is dropped (`profiler.cpp:679-682`). + taken the sample is dropped (`profiler.cpp:781-784`). 3. With the stripe lock held it calls `FlightRecorder::recordEvent(lock_index, ...)`, which writes into `_buf[lock_index]` (`flightRecorder.cpp:2866-2907`). @@ -138,7 +138,7 @@ events write the context snapshot taken when the allocation was tracked - `wallClockEpoch()`, `recordTraceRoot()`, `recordQueueTime()`, `recordDatadogSetting()`, `recordHeapUsage()` and the reference-chain writers use `_buf[lock_index]`; their `Profiler` callers take the stripe lock first - (`profiler.cpp:788-894`, `1854-1866`). + (`profiler.cpp:890-996`, `1854-1866`). - `recordLog()` builds the event in an `alloca` buffer and writes it to the file at once, without a stripe lock (`flightRecorder.cpp:2945-2966`). - `cpuMonitorCycle()` writes `jdk.CPULoad` into `_cpu_monitor_buf`, which @@ -195,12 +195,12 @@ properties and native libraries can be turned off with `NO_SYSTEM_INFO`, ### How `Profiler` drives it -`Profiler::dump()` (`profiler.cpp:2024-2096`) first emits pending +`Profiler::dump()` (`profiler.cpp:2136-2208`) first emits pending reference-chain events under a stripe lock, refreshes libraries and thread names, then runs `_jfr.dump()` inside `rotateDictsAndRun()`. `Profiler::stop()` -runs `_jfr.stop()` the same way (`profiler.cpp:1946`). +runs `_jfr.stop()` the same way (`profiler.cpp:2058`). -`rotateDictsAndRun()` (`profiler.h:216-239`): +`rotateDictsAndRun()` (`profiler.h:245-268`): 1. Blocks profiling signals on the dumping thread (`SignalBlocker`). 2. Rotates `_class_map`, `_string_label_map` and `_context_value_map`, so each @@ -257,7 +257,7 @@ static string table filled by `getId()` (`jfrMetadata.h:114-122`); `Profiler::start()` calls `JfrMetadata::reset()` then `initialize()` with `args._context_attributes`, truncated to `DD_TAGS_CAPACITY`, before -`_jfr.start()` (`profiler.cpp:1657-1665`). `reset()` deletes the tree and +`_jfr.start()` (`profiler.cpp:1760-1768`). `reset()` deletes the tree and re-registers `"root"` as string 0 so the root element's name stays valid (`jfrMetadata.cpp:43-60`). Both require `Profiler::_state_lock` and must not run while engines are active (`jfrMetadata.h:276-284`). @@ -373,7 +373,7 @@ turns this off; it is on by default (`flightRecorder.cpp:1244-1291`, `writeStackTraces()` (`flightRecorder.cpp:1925-1979`) is the only consumer of stored traces. It calls `Profiler::processCallTraces()`, which hands the lambda to `CallTraceStorage::processTraces()`, or an empty set when stack traces are -omitted (`profiler.h:346-354`). Rotation, liveness preservation and the reserved +omitted (`profiler.h:377-385`). Rotation, liveness preservation and the reserved trace IDs are covered in [CallTraceStorage](CallTraceStorage.md). The callback runs inside `processTraces()`, so the `CallTrace` pointers stay valid while it runs. diff --git a/doc/architecture/JvmSupportLayer.md b/doc/architecture/JvmSupportLayer.md index 8a0ecccac3..0378fc3830 100644 --- a/doc/architecture/JvmSupportLayer.md +++ b/doc/architecture/JvmSupportLayer.md @@ -83,11 +83,11 @@ predicates. More detection-dependent setup: - `can_retransform_any_class` is disabled on J9 (`vmEntry.cpp:502`). - The JFR `RequestStackTrace` extension is probed only on HotSpot (`vmEntry.cpp:519-521`). - The profiler installs its SIGSEGV/SIGBUS handlers only on HotSpot and J9 - (`profiler.cpp:1168`). -- On Zing, `GC_TRACES | LAST_JAVA_PC` is forced into `_safe_mode` (`profiler.cpp:1587`). + (`profiler.cpp:1270`). +- On Zing, `GC_TRACES | LAST_JAVA_PC` is forced into `_safe_mode` (`profiler.cpp:1689`). `JVMSupport::initialize()` (`jvmSupport.cpp:32-57`) runs after detection, from -`Profiler::start()`/`Profiler::init()` (`profiler.cpp:1417`, `profiler.cpp:1437`). +`Profiler::start()`/`Profiler::init()` (`profiler.cpp:1519`, `profiler.cpp:1539`). It calls `JVMThread::initialize()`, then checks the `ProfiledThread` TLS key. If either step fails, profiling is disabled. @@ -116,7 +116,7 @@ gets HotSpot declarations, but it should still call only `JVMSupport::*`. Callers in shared code: -- Stack walking calls `walkJavaStack` (`profiler.cpp:717`). +- Stack walking calls `walkJavaStack` (`profiler.cpp:819`). - `stackWalker.cpp:80-82` and `perfEvents_linux.cpp` call `isJitCode` and `canUnwind`. Both are signal-handler paths, and on non-HotSpot VMs they reduce to constant returns. @@ -183,7 +183,7 @@ Caveats: (for example, a build without JFR), `initLazyResolution()` fails, and `initExecution()` falls back to `Partial_loaded` (`jvmSupport.cpp:86-91`). - **`cstack=vm` needs `VMStructs`.** If `VMStructs::hasStackStructs()` is false, - `cstack=vm` is downgraded to DWARF or `no` (`profiler.cpp:1625-1629`). + `cstack=vm` is downgraded to DWARF or `no` (`profiler.cpp:1728-1732`). ### `j9/` @@ -196,7 +196,7 @@ Caveats: - **The ASGCT-or-JVMTI choice is made per engine.** `Profiler::selectCpuEngine` and `selectWallEngine` choose `J9WallClock` (the JVMTI path) when any of these - holds (`profiler.cpp:1274-1285`, `profiler.cpp:1321-1333`): + holds (`profiler.cpp:1376-1387`, `profiler.cpp:1423-1435`): - `J9Support::shouldUseAsgct()` is false. The system property `dd.profiling.ddprof.j9.sampler` set to `jvmti` makes it false (`j9/j9Support.h:108-126`). @@ -223,7 +223,7 @@ behavior is shared code that branches on `VM::isZing()`: - Java stacks are collected with ASGCT (`jvmSupport.cpp:112-117`). - Runtime entries are marked (`vmEntry.cpp:378-379`). -- `_safe_mode` is forced (`profiler.cpp:1587`). +- `_safe_mode` is forced (`profiler.cpp:1689`). Caveat: `JVMThread::nativeThreadId()` sends Zing to `VMThread::nativeThreadId()`. That function returns -1 unless diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 0bce1e95dc..e410e8c110 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1354`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1401`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1436`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1371`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1418`) | Same as above | +| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1453`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,7 +65,7 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1538 +Profiler::start() profiler.cpp:1555 ThreadTablesFreeze unregisterThread() gate closed until the CPU engine sizes its table, at the latest until return @@ -87,30 +87,29 @@ Profiler::start() profiler.cpp:1538 - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1778-1784`). -- The CPU engine is enabled last (`profiler.cpp:1879`). CPU handlers check + false (`wallClock.h:50`, `profiler.cpp:1795-1801`). +- The CPU engine is enabled last (`profiler.cpp:1896`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1899`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1916`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1089`). + only (`profiler.cpp:1106`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1912`). + and tears nothing down (`profiler.cpp:1929`). 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook call that passed the check is still inside an engine. - No pthread-hook call can then arm a timer after `CTimer::stop()`, or still - be using the timer table when the next `start()` resizes it. JVMTI - `ThreadStart` is disabled only after the engines stop, so a Java thread - starting in that window can still arm a timer that nothing deletes. + No thread-hook call (pthread hook or JVMTI callback) can then arm a timer + after `CTimer::stop()`, or still be using the timer table when the next + `start()` resizes it. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2053`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2069`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -118,8 +117,8 @@ each engine's `check()` without starting anything. Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: 1. **JVMTI** `ThreadStart`/`ThreadEnd` reach `Profiler::onThreadStart()`/ - `onThreadEnd()`, which call `_cpu_engine->registerThread(tid)` and - `_wall_engine->registerThread(tid)` (`profiler.cpp:111-112`, `143-144`). + `onThreadEnd()`, which call `Profiler::registerThread()`/ + `unregisterThread()`, so they share the gates described in item 2. When the thread's `ProfiledThread` still exists at thread end, unregistration runs with profiling signals blocked (`blockProfilingForExit()`). @@ -133,14 +132,15 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: `PerfEvents::start()` set via `Profiler::openThreadRegistration()` once their table is sized and before they list threads. `stop()` clears it. `PerfEvents::start()` still closes the fd of every tid missing from its - thread list, including one the hook registered after the list was taken. + thread list, including one the hook registered after the list was taken + (PROF-16359). - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only while `start()` may replace the table (`ThreadTablesFreeze`). It stays set while idle, because `PerfEvents::stop()` keeps per-thread fds open and this call closes the fd of a thread that exits after `stop()`. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` - (`perfEvents_linux.cpp:985`). + (`perfEvents_linux.cpp:987`). Each engine's `start()` also registers every thread that already exists. @@ -164,7 +164,7 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:765-776`). +the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:782-793`). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -175,7 +175,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1354-1399`): +`selectCpuEngine()` (`profiler.cpp:1371-1416`): ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -216,19 +216,19 @@ resolved through `Arguments::cpuSamplerInterval()`. with `SIGEV_THREAD_ID` targeting `tid`, carrying `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:182`). +- **Signal**: `SIGPROF` (`ctimer_linux.cpp:181`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:293-297`). + (`ctimer_linux.cpp:292-296`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above `_max_timers` logs a warning and is not profiled. -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:213-219`). +- **`stop()`** unregisters every slot (`ctimer_linux.cpp:212-218`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:228-284`). It has no ASGCT fallback. + (`ctimer_linux.cpp:227-283`). It has no ASGCT fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -249,13 +249,13 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1056`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1058`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1708`, `1728`). + engine (`profiler.cpp:1725`, `1745`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -290,7 +290,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1401-1434`): on OpenJ9, either +`selectWallEngine()` (`profiler.cpp:1418-1451`): on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -361,7 +361,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:413` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `profiler.cpp:430` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -430,7 +430,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2131`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2147`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From 38394105bfe2fe9daa53ac338f8b2f6aa67eae9a Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 28 Oct 2026 11:23:04 -0400 Subject: [PATCH 12/26] Fix --- ddprof-lib/src/main/cpp/perfEvents.h | 5 ++ ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 38 +++++++----- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 21 ++++++- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 65 ++++++++++++++++++++ doc/architecture/JfrIntegration.md | 2 +- doc/architecture/ProfilingEngines.md | 4 +- 6 files changed, 113 insertions(+), 22 deletions(-) diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h index 560d214352..4c3f761511 100644 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ b/ddprof-lib/src/main/cpp/perfEvents.h @@ -30,6 +30,8 @@ class PerfEventType; class StackContext; class PerfEvents : public Engine { + friend class PerfEventsTestAccessor; + private: static bool _enabled; static int _max_events; @@ -44,6 +46,9 @@ class PerfEvents : public Engine { static u64 readCounter(siginfo_t *siginfo, void *ucontext); // cppcheck-suppress unusedPrivateFunction static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); + // Slot of tid in _events, or NULL when the table is NULL or tid is out of + // range. For readers in signal context (walkKernel(), resetBuffer()). + static PerfEvent *eventFor(int tid); public: Error check(Arguments &args); diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 894b4034b2..e356c5670f 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -969,10 +969,11 @@ Error PerfEvents::start(Arguments &args) { _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); if (_events == NULL) { // _max_events == -1 makes registerThread() treat the engine as not - // started, and the tid bound checks in unregisterThread() and - // resetBuffer() reject every tid, so nothing indexes the NULL table; the - // next start() retries the allocation. Fds left open by an earlier - // recording are not closed here (PROF-16359). + // started; the tid bound check in unregisterThread() and eventFor(), + // used by the signal-path readers walkKernel() and resetBuffer(), reject + // every tid, so nothing indexes the NULL table. The next start() retries + // the allocation. Fds left open by an earlier recording are not closed + // here (PROF-16359). _max_events = -1; return Error("Failed to allocate perf event table"); } @@ -1070,9 +1071,9 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, return 0; } - PerfEvent *event = &_events[tid]; - if (!event->tryLock()) { - return 0; // the event is being destroyed + PerfEvent *event = eventFor(tid); + if (event == NULL || !event->tryLock()) { + return 0; // no table, or the event is being destroyed } int depth = 0; @@ -1158,19 +1159,24 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, return depth; } -void PerfEvents::resetBuffer(int tid) { - // Runs from the SIGPROF handler on every exit, including signals from fds - // left armed by an earlier recording. start() can leave the table NULL - // (_max_events == -1) after a failed allocation, so check the table and the - // bound before indexing it. +// walkKernel() and resetBuffer() run from the SIGPROF handler, including for +// signals from fds left armed by an earlier recording. start() can leave the +// table NULL (_max_events == -1) after a failed allocation, so they reach the +// table only through this check. It does not make a table that start() frees +// and replaces safe to read (PROF-16359). +PerfEvent *PerfEvents::eventFor(int tid) { PerfEvent *events = __atomic_load_n(&_events, __ATOMIC_ACQUIRE); if (events == NULL || tid < 0 || tid >= __atomic_load_n(&_max_events, __ATOMIC_ACQUIRE)) { - return; + return NULL; } - PerfEvent *event = &events[tid]; - if (!event->tryLock()) { - return; // the event is being destroyed + return &events[tid]; +} + +void PerfEvents::resetBuffer(int tid) { + PerfEvent *event = eventFor(tid); + if (event == NULL || !event->tryLock()) { + return; // no table, or the event is being destroyed } struct perf_event_mmap_page *page = event->_page; diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index ab7abfc941..e518175001 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -72,6 +72,22 @@ class ProfilerTestAccessor { Engine *_cpu; Engine *_wall; }; + + // Opens the registration gate for its lifetime and closes it on + // destruction, including when a test assertion returns early. Declare it + // after EngineSwap, so the gate closes before the engines are put back. + class RegistrationGate { + public: + explicit RegistrationGate(Profiler *p) : _p(p) { + p->openThreadRegistration(); + } + ~RegistrationGate() { _p->closeThreadRegistration(); } + RegistrationGate(const RegistrationGate &) = delete; + RegistrationGate &operator=(const RegistrationGate &) = delete; + + private: + Profiler *_p; + }; }; // Polls cond until it holds or a 5 s deadline passes, so a gate regression @@ -289,7 +305,7 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { BlockingEngine cpu; Engine wall; ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); - profiler->openThreadRegistration(); + ProfilerTestAccessor::RegistrationGate gate(profiler); std::thread hook([] { Profiler::registerThread(OS::threadId()); }); bool entered = waitFor([&] { return cpu.entered.load(); }); @@ -344,7 +360,7 @@ TEST(ProfilerEnginesGateTest, CancellationDuringHookCallActsAfterTheCall) { BlockingEngine cpu; Engine wall; ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); - profiler->openThreadRegistration(); + ProfilerTestAccessor::RegistrationGate gate(profiler); survived_cancellation.store(false); pthread_t thread; @@ -355,7 +371,6 @@ TEST(ProfilerEnginesGateTest, CancellationDuringHookCallActsAfterTheCall) { EXPECT_EQ(0, pthread_cancel(thread)); cpu.released.store(true); EXPECT_EQ(0, pthread_join(thread, nullptr)); - ProfilerTestAccessor::closeThreadRegistration(profiler); EXPECT_TRUE(entered); // The hook call ran with cancellation disabled; without that, the request diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index 59bb7c6915..2dae58aa47 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -18,10 +18,12 @@ #ifdef __linux__ +#include "arguments.h" #include "perfEvents.h" #include #include +#include static int compare(int a, int b) { return PerfEvents::compareTids(&a, &b); } @@ -53,4 +55,67 @@ TEST(PerfEventsCompareTidsTest, SortsCreationOrderedTidsWithReuse) { } } +// Swaps PerfEvents' table, bound and ring mode for a test and restores them on +// destruction, including when an assertion returns early. +class PerfEventsTestAccessor { +public: + class TableSwap { + public: + TableSwap(PerfEvent *events, int max_events, Ring ring) + : _events(PerfEvents::_events), _max_events(PerfEvents::_max_events), + _ring(PerfEvents::_ring) { + PerfEvents::_events = events; + PerfEvents::_max_events = max_events; + PerfEvents::_ring = ring; + } + ~TableSwap() { + PerfEvents::_events = _events; + PerfEvents::_max_events = _max_events; + PerfEvents::_ring = _ring; + } + TableSwap(const TableSwap &) = delete; + TableSwap &operator=(const TableSwap &) = delete; + + private: + PerfEvent *_events; + int _max_events; + Ring _ring; + }; +}; + +// After a failed table allocation, start() leaves _events NULL with +// _max_events == -1 while fds from an earlier recording may still fire +// SIGPROF. The signal-path readers must not index the NULL table, and +// unregisterThread() must reject every tid. +TEST(PerfEventsTableTest, SignalPathReadersSkipNullTable) { + PerfEventsTestAccessor::TableSwap swap(nullptr, -1, RING_KERNEL); + const void *callchain[4]; + + for (int tid : {0, 1, 4096}) { + EXPECT_EQ(0, PerfEvents::walkKernel(tid, callchain, 4, nullptr)); + PerfEvents::resetBuffer(tid); + PerfEvents engine; + engine.unregisterThread(tid); + } +} + +// The signal-path readers check the tid against the bound before indexing. +// The table points at an inaccessible page, so any access through it faults. +TEST(PerfEventsTableTest, SignalPathReadersRejectOutOfRangeTid) { + void *page = mmap(nullptr, 1 << 16, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, + -1, 0); + ASSERT_NE(MAP_FAILED, page); + { + PerfEventsTestAccessor::TableSwap swap( + reinterpret_cast(page), 0, RING_KERNEL); + const void *callchain[4]; + + for (int tid : {-1, 0, 1}) { + EXPECT_EQ(0, PerfEvents::walkKernel(tid, callchain, 4, nullptr)); + PerfEvents::resetBuffer(tid); + } + } + munmap(page, 1 << 16); +} + #endif // __linux__ diff --git a/doc/architecture/JfrIntegration.md b/doc/architecture/JfrIntegration.md index 240be4778a..d027e20844 100644 --- a/doc/architecture/JfrIntegration.md +++ b/doc/architecture/JfrIntegration.md @@ -138,7 +138,7 @@ events write the context snapshot taken when the allocation was tracked - `wallClockEpoch()`, `recordTraceRoot()`, `recordQueueTime()`, `recordDatadogSetting()`, `recordHeapUsage()` and the reference-chain writers use `_buf[lock_index]`; their `Profiler` callers take the stripe lock first - (`profiler.cpp:890-996`, `1854-1866`). + (`profiler.cpp:890-996`, `1966-1978`). - `recordLog()` builds the event in an `alloca` buffer and writes it to the file at once, without a stripe lock (`flightRecorder.cpp:2945-2966`). - `cpuMonitorCycle()` writes `jdk.CPULoad` into `_cpu_monitor_buf`, which diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index e410e8c110..d4a2cfb173 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -140,7 +140,7 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: and this call closes the fd of a thread that exits after `stop()`. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` - (`perfEvents_linux.cpp:987`). + (`perfEvents_linux.cpp:988`). Each engine's `start()` also registers every thread that already exists. @@ -249,7 +249,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1058`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1059`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted From e4ecdd192632556fc247fd4ef6b50d257861b76b Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 28 Oct 2026 12:09:00 -0400 Subject: [PATCH 13/26] PROF-16213: Document the thread-registration contract in AGENTS.md The gated, in-flight-counted registration path introduced by this branch is a load-bearing invariant for any code that touches per-thread engine tables, but it was documented only in ProfilingEngines.md. - Add a "Thread registration" critical invariant to AGENTS.md: reach the engines only through Profiler::registerThread()/unregisterThread(); size a per-thread table, then openThreadRegistration(), then list threads; free or replace a table only while its gate is closed; restore the cancellation state outside destructor-bearing scopes. - Mention the gates in the AGENTS.md subsystem table and the doc/README.md index entry. - Describe the in-flight count and the cancellation rule in ProfilingEngines.md#thread-registration. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 21 ++++++++++++++++++++- doc/README.md | 2 +- doc/architecture/ProfilingEngines.md | 16 ++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a574e785fa..fb428bbecd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -146,7 +146,7 @@ before working on a subsystem. | String dictionary (arena chunks, rotation/clear protocols) | [doc/architecture/StringDictionary.md](doc/architecture/StringDictionary.md) | | RefCountGuard (lock-free RAII reader-drain before reclamation) | [doc/architecture/RefCountGuard.md](doc/architecture/RefCountGuard.md) | | TLS context (thread-local tracing context, distributed tracing) | [doc/architecture/TLSContext.md](doc/architecture/TLSContext.md) | -| Profiling engines (CPU, wall-clock, allocation, native memory, liveness) | [doc/architecture/ProfilingEngines.md](doc/architecture/ProfilingEngines.md) | +| Profiling engines (CPU, wall-clock, allocation, native memory, liveness; thread-registration gates) | [doc/architecture/ProfilingEngines.md](doc/architecture/ProfilingEngines.md) | | JFR integration (FlightRecorder, metadata, constant pools, buffers) | [doc/architecture/JfrIntegration.md](doc/architecture/JfrIntegration.md) | | Live-heap reference chains (implementation, signal scheduling, design) | [doc/architecture/LiveHeapReferenceChainsImplementation.md](doc/architecture/LiveHeapReferenceChainsImplementation.md), [doc/architecture/ReferenceChainsSignalsExplained.md](doc/architecture/ReferenceChainsSignalsExplained.md), [doc/ReferenceChainsDesign.md](doc/ReferenceChainsDesign.md), [doc/ReferenceChainsCollectionSummary.md](doc/ReferenceChainsCollectionSummary.md) | | Native memory profiling (malloc/free interception, sampling) | [doc/architecture/NativeMemoryProfiling.md](doc/architecture/NativeMemoryProfiling.md) | @@ -272,6 +272,25 @@ Otherwise a signal can sample a half-torn-down thread, or re-acquire a thread that keeps running silently stops profiling that thread — use `SignalBlocker` for a temporary window. +**Thread registration.** The `pthread_create` hook and the JVMTI `ThreadStart`/ +`ThreadEnd` callbacks run without `_state_lock` and can overlap `Profiler::start()`/ +`stop()`, while engines size, replace or tear down their per-thread tables (`CTimer`'s +`_timers`, `PerfEvents`' `_events`). So: +- reach the engines only through `Profiler::registerThread()`/`unregisterThread()`, + never `_cpu_engine->registerThread()` directly. These calls check a gate + (`_engines_ready` / `_thread_tables_stable`), count themselves in flight + (`HookCallScope`), and run with thread cancellation disabled; +- an engine with a per-thread table sizes it first, then calls + `Profiler::openThreadRegistration()`, then lists the existing threads. Free or + replace a table only while its gate is closed: closing a gate + (`closeThreadRegistration()`, `ThreadTablesFreeze`) waits until no call is inside an + engine; +- restore the cancellation state outside every destructor-bearing scope, never from a + destructor: current glibc acts on a pending asynchronous cancellation as soon as it + is re-enabled, and unwinding out of an implicitly `noexcept` destructor terminates. + +Details: [ProfilingEngines.md](doc/architecture/ProfilingEngines.md#thread-registration). + **JVMTI callbacks.** Callbacks run on whichever JVM thread triggers the event — including compiler, VM and GC threads that never went through our `pthread_create` hook — so call `ProfiledThread::initCurrentThreadSignalSafe()` first. diff --git a/doc/README.md b/doc/README.md index 39d51ddf17..a6b20560f4 100644 --- a/doc/README.md +++ b/doc/README.md @@ -22,7 +22,7 @@ All documentation files use **PascalCase** naming (e.g., `BuildSystemGuide.md`). - [JfrIntegration](architecture/JfrIntegration.md) - Native JFR writer: recordings, buffers, metadata, constant pools - [LiveHeapReferenceChainsImplementation](architecture/LiveHeapReferenceChainsImplementation.md) - Current reference-chain implementation - [NativeMemoryProfiling](architecture/NativeMemoryProfiling.md) - malloc interception and allocation sampling -- [ProfilingEngines](architecture/ProfilingEngines.md) - CPU, wall-clock, allocation, native-memory and liveness engines +- [ProfilingEngines](architecture/ProfilingEngines.md) - CPU, wall-clock, allocation, native-memory and liveness engines; thread-registration gates and in-flight drain - [RefCountGuard](architecture/RefCountGuard.md) - Lock-free RAII reference-counting primitive used to drain readers before resource reclamation - [ReferenceChainsSignalsExplained](architecture/ReferenceChainsSignalsExplained.md) - Reference-chain scheduling and cost control - [JvmSupportLayer](architecture/JvmSupportLayer.md) - JVM-implementation abstraction (HotSpot, J9, Zing) diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index d4a2cfb173..833931c595 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -142,6 +142,22 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:988`). +Every `Profiler::registerThread()`/`unregisterThread()` call counts itself in +flight (`HookCallScope`) for as long as it may be inside an engine. Closing a gate +(`closeThreadRegistration()` in `stop()`, `ThreadTablesFreeze` in `start()`) waits +for that count to reach zero, so no call is inside an engine while it is torn down +or its table is replaced. The gate flags and the counter use `seq_cst` on both +sides: either a call sees the gate closed, or the closer sees the call and waits. + +Thread cancellation is disabled for the length of each call, because the engine +calls reach cancellation points (`close()`, `Log::warn()`). Acting on a +cancellation there would unwind through `~HookCallScope()`, which aborts with the +statically linked libgcc, or, on musl, skip it and leave the count raised. The +state is restored by a plain call made after every destructor-bearing scope of the +call, not by a guard. Current glibc acts on a pending asynchronous cancellation as +soon as it is re-enabled, and unwinding out of an implicitly `noexcept` destructor +terminates the process. + Each engine's `start()` also registers every thread that already exists. ## Common Signal-Handler Contract From 08375d323bd1dc5ff42b308860390cb507c91e81 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Wed, 28 Oct 2026 20:17:53 -0400 Subject: [PATCH 14/26] PROF-16213: Enable the CPU engine only if it started Profiler::start() called _cpu_engine->enableEvents(true) whenever any mode activated. If the CPU engine's start() failed (for example a PerfEvents table allocation failure on a restart after pid_max changed) and another mode started, the engine was enabled anyway, and perf fds left armed by the previous recording resumed emitting CPU samples into a recording whose CPU engine had failed. Enable the CPU engine only when EM_CPU activated. Retiring the stale PerfEvents fds themselves remains tracked in PROF-16359. Update the ProfilingEngines.md lifecycle notes and recompute the shifted profiler.cpp line references in ProfilingEngines.md and JfrIntegration.md. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/profiler.cpp | 8 +++++++- doc/architecture/JfrIntegration.md | 6 +++--- doc/architecture/ProfilingEngines.md | 16 +++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index ffee2fbac4..a9645c4e8a 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -1893,7 +1893,13 @@ Error Profiler::start(Arguments &args, bool reset) { // re-check _enabled on every signal, so enabling here (rather than before // start()) does not lose samples. // Paired with drainInflight() on the stop side. - _cpu_engine->enableEvents(true); + // Only when the CPU engine itself started: another mode may have activated + // after it failed, and fds or timers left armed by an earlier recording + // (PerfEvents keeps its fds across stop()) would otherwise emit CPU samples + // into this recording. + if (activated & EM_CPU) { + _cpu_engine->enableEvents(true); + } _state.store(RUNNING, std::memory_order_release); _start_time = time(NULL); diff --git a/doc/architecture/JfrIntegration.md b/doc/architecture/JfrIntegration.md index d027e20844..402c5a0158 100644 --- a/doc/architecture/JfrIntegration.md +++ b/doc/architecture/JfrIntegration.md @@ -138,7 +138,7 @@ events write the context snapshot taken when the allocation was tracked - `wallClockEpoch()`, `recordTraceRoot()`, `recordQueueTime()`, `recordDatadogSetting()`, `recordHeapUsage()` and the reference-chain writers use `_buf[lock_index]`; their `Profiler` callers take the stripe lock first - (`profiler.cpp:890-996`, `1966-1978`). + (`profiler.cpp:890-996`, `1972-1984`). - `recordLog()` builds the event in an `alloca` buffer and writes it to the file at once, without a stripe lock (`flightRecorder.cpp:2945-2966`). - `cpuMonitorCycle()` writes `jdk.CPULoad` into `_cpu_monitor_buf`, which @@ -195,10 +195,10 @@ properties and native libraries can be turned off with `NO_SYSTEM_INFO`, ### How `Profiler` drives it -`Profiler::dump()` (`profiler.cpp:2136-2208`) first emits pending +`Profiler::dump()` (`profiler.cpp:2142-2214`) first emits pending reference-chain events under a stripe lock, refreshes libraries and thread names, then runs `_jfr.dump()` inside `rotateDictsAndRun()`. `Profiler::stop()` -runs `_jfr.stop()` the same way (`profiler.cpp:2058`). +runs `_jfr.stop()` the same way (`profiler.cpp:2064`). `rotateDictsAndRun()` (`profiler.h:245-268`): diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 833931c595..9eb60cd6aa 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -82,24 +82,26 @@ Profiler::start() profiler.cpp:1555 NativeSocketSampler::start() (EM_NATIVESOCKET) if anything activated: switchThreadEvents(JVMTI_ENABLE) - _cpu_engine->enableEvents(true) LAST + _cpu_engine->enableEvents(true) LAST, only if EM_CPU activated ``` - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is false (`wallClock.h:50`, `profiler.cpp:1795-1801`). -- The CPU engine is enabled last (`profiler.cpp:1896`). CPU handlers check - `_enabled` on every signal, so nothing is lost by enabling late. +- The CPU engine is enabled last (`profiler.cpp:1901`). CPU handlers check + `_enabled` on every signal, so nothing is lost by enabling late. It is not + enabled when its own `start()` failed, even if another mode activated, so fds + or timers left armed by an earlier recording cannot emit CPU samples. - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1916`) runs in reverse: +`Profiler::stop()` (`profiler.cpp:1922`) runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines only (`profiler.cpp:1106`). 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1929`). + and tears nothing down (`profiler.cpp:1935`). 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook call that passed the check is still inside an engine. No thread-hook call (pthread hook or JVMTI callback) can then arm a timer @@ -109,7 +111,7 @@ Profiler::start() profiler.cpp:1555 `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2069`) runs the same selection and calls +`Profiler::check()` (`profiler.cpp:2075`) runs the same selection and calls each engine's `check()` without starting anything. ### Thread registration @@ -446,7 +448,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2147`). The +- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2153`). The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage From a24bc50cc942fcfb9fe2921f4ef9b5a4c6d95eac Mon Sep 17 00:00:00 2001 From: Jaroslav Bachorik Date: Thu, 29 Oct 2026 15:21:07 +0100 Subject: [PATCH 15/26] sphinx: address review feedback on PR #912 --- AGENTS.md | 6 +- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 4 +- ddprof-lib/src/main/cpp/nativeMem.h | 22 +- ddprof-lib/src/main/cpp/perfEvents.h | 25 +- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 150 +++++++-- ddprof-lib/src/main/cpp/profiler.cpp | 171 +++++++--- ddprof-lib/src/main/cpp/profiler.h | 81 ++++- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 332 ++++++++++++++++++- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 243 ++++++++++++++ doc/architecture/ProfilingEngines.md | 93 ++++-- doc/reference/ProfilerMemoryRequirements.md | 7 +- 11 files changed, 996 insertions(+), 138 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fb428bbecd..76af8d151c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -284,7 +284,11 @@ for a temporary window. `Profiler::openThreadRegistration()`, then lists the existing threads. Free or replace a table only while its gate is closed: closing a gate (`closeThreadRegistration()`, `ThreadTablesFreeze`) waits until no call is inside an - engine; + engine. The wait is bounded; when it reports failure, tear nothing down, replace no + table and return a retryable error. `Profiler::startCpuEngine()` opens the + registration gate once any CPU engine started, so the gate is open iff it did; +- an engine's `stop()` must tolerate a concurrent `unregisterThread()`: `stop()` closes + only the registration gate, and unregistration stays open while the engines stop; - restore the cancellation state outside every destructor-bearing scope, never from a destructor: current glibc acts on a pending asynchronous cancellation as soon as it is re-enabled, and unwinding out of an implicitly `noexcept` destructor terminates. diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index ee133ebcaf..d674d844a1 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -55,7 +55,7 @@ bool CTimer::_enabled = false; int CTimer::_signal; int CTimer::registerThread(int tid) { - if (tid >= _max_timers) { + if (tid < 0 || tid >= _max_timers) { Log::warn("tid[%d] > pid_max[%d]. Restart profiler after changing pid_max", tid, _max_timers); return -1; @@ -120,7 +120,7 @@ int CTimer::registerThread(int tid) { } void CTimer::unregisterThread(int tid) { - if (tid >= _max_timers) { + if (tid < 0 || tid >= _max_timers) { return; } // Atomic acquire to avoid possible leak when unregistering diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 8545e1dd1c..3e4886cf37 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -14,10 +14,19 @@ // per-category live byte gauges partition the total: sum(category) == total, // with no double counting. // -// These gauges track memory that is actually touched (and therefore resident), -// not address space that has merely been reserved. NM_CALLTRACE in particular -// counts the bytes LinearAllocator::alloc() has bump-allocated, NOT the -// capacity of the 8 MiB chunks backing them -- see linearAllocator.cpp. +// With the exceptions below, these gauges track memory that is actually touched +// (and therefore resident), not address space that has merely been reserved. +// NM_CALLTRACE in particular counts the bytes LinearAllocator::alloc() has +// bump-allocated, NOT the capacity of the 8 MiB chunks backing them -- see +// linearAllocator.cpp. +// +// Exceptions: the tid-indexed tables of CTimer (NM_CTIMER, +// pid_max * sizeof(int)) and PerfEvents (part of NM_PERF, +// pid_max * sizeof(PerfEvent), two cache lines per slot) are counted at their +// full calloc'd size. That memory is mostly untouched address space -- only the +// pages holding live tids become resident -- so for these two categories the +// gauge overstates RSS and should be read as an upper bound. NM_PERF's +// per-thread ring pages (2 pages each) are touched and resident. // // Consequently NM_CALLTRACE is no longer an "arena reserved" figure with // CALLTRACE_STORAGE_BYTES as its used slice: both now count touched bytes, and @@ -25,6 +34,11 @@ // not chunk slack. Do not compute arena waste as the difference between them. // DICTIONARY_ARENA_WASTE_BYTES remains a genuine nested waste figure for the // DICTIONARY arena. None of these nested counters are summed in here. +// +// The exported counter names (native_mem_{live,avg,max,chunk_overhead}_bytes. +// ) derive from this table, and the set grows when a category is +// added (ctimer is the newest), so consumers should match native_mem_*_bytes. +// counters by prefix rather than hard-code the category list. #define DD_NATIVE_MEM_CATEGORY_TABLE(X) \ X(CALLTRACE, "calltrace") \ X(DICTIONARY, "dictionary") \ diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h index 4c3f761511..cd77eeb5dd 100644 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ b/ddprof-lib/src/main/cpp/perfEvents.h @@ -46,10 +46,31 @@ class PerfEvents : public Engine { static u64 readCounter(siginfo_t *siginfo, void *ucontext); // cppcheck-suppress unusedPrivateFunction static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - // Slot of tid in _events, or NULL when the table is NULL or tid is out of - // range. For readers in signal context (walkKernel(), resetBuffer()). + // Slot of tid in _events, or NULL when the table is NULL, being replaced, + // or tid is out of range. For readers in signal context (walkKernel(), + // resetBuffer()). static PerfEvent *eventFor(int tid); + // Generation of the (_events, _max_events) pair; odd while resizeEvents() + // replaces the table. eventFor() accepts a pair only from one even + // generation. + static unsigned int _events_gen; + static void beginTableUpdate(); + static void endTableUpdate(); + // Replaces the tid-indexed table with one of max_events slots, after + // releasing every fd and ring held by the old table. No-op when the size is + // unchanged. On allocation failure leaves _events == NULL and + // _max_events == -1 and returns false. Called only from start(), with the + // Profiler's registration gates closed. + bool resizeEvents(int max_events); + +#ifdef UNIT_TEST + // Stores fd and ring page into slot tid of the current table (test only). + static void setSlotForTest(int tid, int fd, void *page); + // Reads back the fd of slot tid of the current table (test only). + static int slotFdForTest(int tid); +#endif + public: Error check(Arguments &args); Error start(Arguments &args); diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index e356c5670f..b815166409 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -574,6 +574,7 @@ class PerfEvent : public SpinLock { bool PerfEvents::_enabled = false; int PerfEvents::_max_events = -1; PerfEvent *PerfEvents::_events = NULL; +unsigned int PerfEvents::_events_gen = 0; PerfEventType *PerfEvents::_event_type = NULL; long PerfEvents::_interval; Ring PerfEvents::_ring; @@ -594,7 +595,7 @@ int PerfEvents::registerThread(int tid) { // It hasn't been started return 0; } - if (tid >= _max_events) { + if (tid < 0 || tid >= _max_events) { Log::warn("tid[%d] > pid_max[%d]. Restart profiler after changing pid_max", tid, _max_events); return -1; @@ -712,7 +713,7 @@ int PerfEvents::registerThread(int tid) { } void PerfEvents::unregisterThread(int tid) { - if (tid >= _max_events) { + if (tid < 0 || tid >= _max_events) { return; } @@ -911,6 +912,78 @@ Error PerfEvents::check(Arguments &args) { return Error::OK; } +void PerfEvents::beginTableUpdate() { + __atomic_store_n(&_events_gen, + __atomic_load_n(&_events_gen, __ATOMIC_RELAXED) + 1, + __ATOMIC_RELAXED); + // The odd generation is visible before any of the table stores. + __atomic_thread_fence(__ATOMIC_RELEASE); +} + +void PerfEvents::endTableUpdate() { + __atomic_store_n(&_events_gen, + __atomic_load_n(&_events_gen, __ATOMIC_RELAXED) + 1, + __ATOMIC_RELEASE); +} + +// The only writer of _events and _max_events outside tests. start() runs under +// the Profiler's ThreadTablesFreeze with the registration gate closed, so the +// hook stays off the table; only the signal-path readers (eventFor()) run +// concurrently, and the generation bracket lets them reject a pair read while +// the table is replaced. +bool PerfEvents::resizeEvents(int max_events) { + if (max_events == _max_events) { + // Keeps the fds across stop()/start() when pid_max is unchanged. + return true; + } + // The table is the only record of which fd and ring belong to which tid. + // Release them while it still exists, otherwise neither this start() nor a + // later one can close them; start()'s sweep then re-registers the live + // threads into the new table. unregisterThread() excludes the signal-path + // readers with the slot lock, and is a no-op for an empty slot (fd 0 or -1, + // no page). The walk touches every slot, but runs only when pid_max changes + // or after a failed allocation. + if (_events != NULL) { + for (int tid = 0; tid < _max_events; tid++) { + PerfEvents::unregisterThread(tid); + } + } + PerfEvent *old_events = _events; + int old_max = _max_events; + beginTableUpdate(); + __atomic_store_n(&_events, (PerfEvent *)NULL, __ATOMIC_RELAXED); + __atomic_store_n(&_max_events, -1, __ATOMIC_RELAXED); + free(old_events); + // Record the decrement after the free, consistent with the other decrement + // sites; the guard keeps an earlier failed allocation from being released. + if (old_events != NULL && old_max > 0) { + NativeMem::record(NM_PERF, + -(long long)((size_t)old_max * sizeof(PerfEvent))); + } + // On failure the pair stays NULL / -1: registerThread() treats the engine as + // not started, and the tid bound checks in unregisterThread() and eventFor() + // reject every tid, so nothing indexes the NULL table. The next start() + // retries the allocation. + PerfEvent *events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); + if (events != NULL) { + __atomic_store_n(&_events, events, __ATOMIC_RELAXED); + __atomic_store_n(&_max_events, max_events, __ATOMIC_RELAXED); + NativeMem::record(NM_PERF, + (long long)((size_t)max_events * sizeof(PerfEvent))); + } + endTableUpdate(); + return events != NULL; +} + +#ifdef UNIT_TEST +void PerfEvents::setSlotForTest(int tid, int fd, void *page) { + _events[tid]._fd = fd; + _events[tid]._page = (struct perf_event_mmap_page *)page; +} + +int PerfEvents::slotFdForTest(int tid) { return _events[tid]._fd; } +#endif + Error PerfEvents::start(Arguments &args) { _event_type = PerfEventType::forName(args._event == NULL ? EVENT_CPU : args._event); @@ -954,32 +1027,12 @@ Error PerfEvents::start(Arguments &args) { (_ring != RING_USER || _cstack == CSTACK_DEFAULT || _cstack == CSTACK_LBR); - int max_events = OS::getMaxThreadId(); - if (max_events != _max_events) { - // Account the per-thread PerfEvent array under NM_PERF, alongside the ring - // mappings. Guard on non-null so a prior calloc failure isn't mis-accounted. - // Record the decrement after the free (consistent with the other decrement - // sites); capture the old size first since _max_events is overwritten below. - long long old_bytes = - (_events != NULL) ? (long long)((size_t)_max_events * sizeof(PerfEvent)) : 0; - free(_events); - if (old_bytes > 0) { - NativeMem::record(NM_PERF, -old_bytes); - } - _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); - if (_events == NULL) { - // _max_events == -1 makes registerThread() treat the engine as not - // started; the tid bound check in unregisterThread() and eventFor(), - // used by the signal-path readers walkKernel() and resetBuffer(), reject - // every tid, so nothing indexes the NULL table. The next start() retries - // the allocation. Fds left open by an earlier recording are not closed - // here (PROF-16359). - _max_events = -1; - return Error("Failed to allocate perf event table"); - } - _max_events = max_events; - NativeMem::record(NM_PERF, - (long long)((size_t)max_events * sizeof(PerfEvent))); + // Account the per-thread PerfEvent array under NM_PERF, alongside the ring + // mappings. resizeEvents() closes the fds of the old table before freeing + // it; on allocation failure the table stays empty and the next start() + // retries. + if (!resizeEvents(OS::getMaxThreadId())) { + return Error("Failed to allocate perf event table"); } OS::installSignalHandler(SIGPROF, signalHandler); @@ -1036,6 +1089,25 @@ Error PerfEvents::start(Arguments &args) { free(threads); + if (err == 0) { + // The sweep above closes the fd of every tid missing from the snapshot, + // including a thread the hook registered after the snapshot was taken. + // Such a thread was already live, so a second listing sees it; + // registerThread() skips tids that kept their fd and its 0 -> -1 CAS + // resolves races with the hook. Threads started after the sweep are left + // to the hook. Best effort: never fails start() (PROF-16359). + ThreadList *rearm = OS::listThreads(); + while (rearm->hasNext()) { + int tid = rearm->next(); + int rc = registerThread(tid); + if (rc != 0 && rc != ESRCH && rc != -1) { + Log::debug("perfEvents: re-registering TID %d failed: %s", tid, + strerror(rc)); + } + } + delete rearm; + } + if (err != 0) { *_pthread_entry = (void *)pthread_setspecific; Profiler::instance()->switchThreadEvents(JVMTI_DISABLE); @@ -1162,12 +1234,24 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, // walkKernel() and resetBuffer() run from the SIGPROF handler, including for // signals from fds left armed by an earlier recording. start() can leave the // table NULL (_max_events == -1) after a failed allocation, so they reach the -// table only through this check. It does not make a table that start() frees -// and replaces safe to read (PROF-16359). +// table only through this check. The generation check makes the bound always +// belong to the table it validates, including while resizeEvents() replaces +// the table; the reader never spins, it returns NULL instead, so a sample on +// the start() thread inside that window loses its kernel frames. A slot +// pointer returned before resizeEvents() frees the table is still unsafe to +// use afterwards (PROF-16359). PerfEvent *PerfEvents::eventFor(int tid) { - PerfEvent *events = __atomic_load_n(&_events, __ATOMIC_ACQUIRE); - if (events == NULL || tid < 0 || - tid >= __atomic_load_n(&_max_events, __ATOMIC_ACQUIRE)) { + unsigned int gen = __atomic_load_n(&_events_gen, __ATOMIC_ACQUIRE); + if (gen & 1) { + return NULL; // the table is being replaced + } + PerfEvent *events = __atomic_load_n(&_events, __ATOMIC_RELAXED); + int max_events = __atomic_load_n(&_max_events, __ATOMIC_RELAXED); + __atomic_thread_fence(__ATOMIC_ACQUIRE); + if (__atomic_load_n(&_events_gen, __ATOMIC_RELAXED) != gen) { + return NULL; // the table changed while it was read + } + if (events == NULL || tid < 0 || tid >= max_events) { return NULL; } return &events[tid]; diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index a9645c4e8a..540d6b3100 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -61,11 +62,17 @@ #include #include #include +#include #include +// Budget for waitForHookCalls(). Hook calls do a few syscalls per engine, so +// a full second means a call is stuck (or the in-flight count leaked). +static const long HOOK_CALLS_DRAIN_TIMEOUT_NS = 1000000000L; + // The instance is not deleted on purpose, since profiler structures // can be still accessed concurrently during VM termination Profiler *const Profiler::_instance = new Profiler(); +long long Profiler::_hook_drain_timeout_ns = HOOK_CALLS_DRAIN_TIMEOUT_NS; volatile bool Profiler::_signals_initialized = false; volatile bool Profiler::_need_JDK_8313796_workaround = true; @@ -170,7 +177,9 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // and the calls that already passed it waited for. The increment and the gate // load in the caller are seq_cst, pairing with the seq_cst gate store and // counter load on the closing side: either the hook sees the gate closed, or -// the closer sees the call in flight and waits for it. +// the closer sees the call in flight and waits for it. registerThreadInEngines() +// skips the scope when an acquire pre-check already sees the registration gate +// closed; the decision that matters is still the seq_cst pair. class HookCallScope { public: explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { @@ -184,45 +193,54 @@ class HookCallScope { std::atomic &_inflight; }; -// Keeps Profiler::unregisterThread() off the engines while start() selects -// the CPU engine and lets it replace its per-thread table. The gate is set -// again by openThreadRegistration() once the table is sized, and at the latest -// when start() returns. An exit skipped in this window is cleaned up only if -// the thread is gone when the engine lists threads: CTimer::stop() deleted -// every timer, and PerfEvents::start() closes the fd of every listed-missing -// tid. A thread that is still listed while it exits is registered by start() -// and its slot stays occupied after it dies (PROF-16359). -class Profiler::ThreadTablesFreeze { -public: - explicit ThreadTablesFreeze(Profiler *profiler) : _profiler(profiler) { - _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); - _profiler->waitForHookCalls(); - } - ~ThreadTablesFreeze() { - _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); - } - ThreadTablesFreeze(const ThreadTablesFreeze &) = delete; - ThreadTablesFreeze &operator=(const ThreadTablesFreeze &) = delete; - -private: - Profiler *_profiler; -}; - void Profiler::openThreadRegistration() { _thread_tables_stable.store(true, std::memory_order_seq_cst); _engines_ready.store(true, std::memory_order_seq_cst); } -void Profiler::closeThreadRegistration() { +bool Profiler::closeThreadRegistration() { _engines_ready.store(false, std::memory_order_seq_cst); - waitForHookCalls(); -} + return waitForHookCalls(); +} + +bool Profiler::waitForHookCalls() { + // Hook calls only do a few syscalls per engine, so the wait is short. It is + // bounded so that a call that never leaves (or a leaked in-flight count) + // fails start()/stop() with a retryable error instead of wedging them under + // _state_lock. clock_gettime() rather than OS::nanotime(): see + // SamplerPerf::primeClock() in start(). + if (_hook_calls_inflight.load(std::memory_order_seq_cst) == 0) { + return true; + } + long long budget = _hook_drain_timeout_ns; + struct timespec deadline; + if (clock_gettime(CLOCK_MONOTONIC, &deadline) != 0) { + Log::warn("Profiler: clock_gettime(CLOCK_MONOTONIC) failed (errno=%d) " + "waiting for thread hook calls", errno); + return false; + } + // Division/modulo rather than a single carry: the budget may exceed 2 s. + long long nsec = (long long)deadline.tv_nsec + budget; + deadline.tv_sec += (time_t)(nsec / 1000000000LL); + deadline.tv_nsec = (long)(nsec % 1000000000LL); -void Profiler::waitForHookCalls() { - // Hook calls only do a few syscalls per engine, so the wait is short. while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { + struct timespec now; + if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) { + Log::warn("Profiler: clock_gettime(CLOCK_MONOTONIC) failed (errno=%d) " + "waiting for thread hook calls", errno); + return false; + } + if (now.tv_sec > deadline.tv_sec || + (now.tv_sec == deadline.tv_sec && now.tv_nsec >= deadline.tv_nsec)) { + int remaining = _hook_calls_inflight.load(std::memory_order_seq_cst); + Log::warn("Profiler: timed out after %ldms waiting for %d in-flight " + "thread hook call(s)", (long)(budget / 1000000LL), remaining); + return false; + } sched_yield(); } + return true; } // Thread cancellation is disabled for the whole hook call. The engine calls @@ -254,6 +272,13 @@ void Profiler::unregisterThread(int tid) { } int Profiler::registerThreadInEngines(int tid) { + // Advisory pre-check: skips the shared counter RMW while idle. A stale + // "closed" read equals a call made just before the gate opened, which the + // CPU engine's start() covers by listing threads after opening it; a stale + // "open" read falls through to the seq_cst re-check below. + if (!_instance->_engines_ready.load(std::memory_order_acquire)) { + return -1; // not registered, as Engine::registerThread() reports it + } HookCallScope scope(_instance->_hook_calls_inflight); // Threads created before the gate opens are picked up by the CPU engine's // start(), which opens the gate before it lists the existing threads. @@ -1108,6 +1133,39 @@ void Profiler::disableEngines() { _wall_engine->enableEvents(false); } +// The registration gate is open for a recording only if the CPU engine +// started. An engine that failed or was not started must never see the hook: +// its registerThread() would arm a timer or open a perf fd for it. Engines with +// a per-thread table (CTimer, PerfEvents) open the gate themselves before they +// list threads, so opening it again here does nothing for them. For engines +// without a table (ITimer, ITimerJvmti, J9) it opens the gate, so registration +// follows the profiler lifecycle rather than the engine choice. No wall engine +// overrides Engine::registerThread(), so a recording whose CPU engine did not +// start loses nothing with the gate closed. +bool Profiler::startCpuEngine(Arguments &args) { + Error error = _cpu_engine->start(args); + if (error) { + // The engine may have opened the gate before failing. A call that is + // still in flight after the wait budget is waited for again by the next + // start()'s ThreadTablesFreeze before any table is replaced. + (void)closeThreadRegistration(); + Log::warn("%s", error.message()); + return false; + } + openThreadRegistration(); + return true; +} + +void Profiler::enableCpuEngine(int activated) { + // Only when the CPU engine itself started: another mode may have activated + // after it failed, and fds or timers left armed by an earlier recording + // (PerfEvents keeps its fds across stop()) would otherwise emit CPU samples + // into this recording. + if (activated & EM_CPU) { + _cpu_engine->enableEvents(true); + } +} + void Profiler::segvHandler(int signo, siginfo_t *siginfo, void *ucontext) { // J9 installs a SIGSEGV handler that uses siglongjmp() to recover from // null-pointer-check faults during normal Java execution. When we chain to @@ -1707,6 +1765,12 @@ Error Profiler::start(Arguments &args, bool reset) { } ThreadTablesFreeze tables_freeze(this); + if (!tables_freeze.drained()) { + // Nothing selected or replaced yet; the freeze destructor reopens + // unregistration and _engines_ready stays closed. A later start() waits + // again. Retryable, like the earlier error returns in this function. + return Error("thread hook calls did not drain; retry start()"); + } _cpu_engine = selectCpuEngine(args); _wall_engine = selectWallEngine(args); _cstack = args._cstack; @@ -1780,17 +1844,12 @@ Error Profiler::start(Arguments &args, bool reset) { // can fire. SamplerPerf::primeClock(); + // error is Error::OK here: every earlier failure returned. A CPU engine + // failure is recoverable and logged by startCpuEngine(). int activated = 0; - if ((_event_mask & EM_CPU) && _cpu_engine != &noop_engine) { - error = _cpu_engine->start(args); - if (error) { - // The engine may have opened the gate before failing. - closeThreadRegistration(); - Log::warn("%s", error.message()); - error = Error::OK; // recoverable - } else { - activated |= EM_CPU; - } + if ((_event_mask & EM_CPU) && _cpu_engine != &noop_engine && + startCpuEngine(args)) { + activated |= EM_CPU; } if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { // Enable wall clock BEFORE starting its pthread: timerLoopCommon() checks @@ -1892,14 +1951,9 @@ Error Profiler::start(Arguments &args, bool reset) { // ever sees finds the engine fully wired. Unlike wall clock, CPU handlers // re-check _enabled on every signal, so enabling here (rather than before // start()) does not lose samples. - // Paired with drainInflight() on the stop side. - // Only when the CPU engine itself started: another mode may have activated - // after it failed, and fds or timers left armed by an earlier recording - // (PerfEvents keeps its fds across stop()) would otherwise emit CPU samples - // into this recording. - if (activated & EM_CPU) { - _cpu_engine->enableEvents(true); - } + // Paired with drainInflight() on the stop side. enableCpuEngine() enables + // it only if the CPU engine itself started. + enableCpuEngine(activated); _state.store(RUNNING, std::memory_order_release); _start_time = time(NULL); @@ -1942,11 +1996,22 @@ Error Profiler::stop() { } // Close the registration gate and wait for calls already past it, so no - // thread arms a timer after CTimer::stop() has unregistered them all, and - // none is still inside the timer table when the next start() resizes it. - // This covers the JVMTI ThreadStart/ThreadEnd callbacks too: they go through - // the same gate, and their events are disabled only after the engines stop. - closeThreadRegistration(); + // thread arms a timer or opens an fd after the engines below stop, and + // none is still inside a table when the next start() resizes it. JVMTI + // ThreadStart goes through the same gate. Only registration is closed: + // JVMTI ThreadEnd and native thread exits may still reach unregisterThread() + // while the engines stop. That is safe per engine: CTimer::stop() and + // CTimer::unregisterThread() each release a slot with a per-slot CAS and + // neither frees the table; PerfEvents::stop() is a no-op and the slot is + // released under its fd CAS and slot lock; wall engines and ITimer have no + // unregisterThread(). Only start() replaces tables, under + // ThreadTablesFreeze. Unregistration stays open so PerfEvents can close the + // fd of a thread that exits after stop(). + if (!closeThreadRegistration()) { + // Same retry contract as the SignalInflight timeout above: state stays + // RUNNING, no engine stop has run, and a retry re-runs every step. + return Error("thread hook calls did not drain; teardown skipped, retry stop()"); + } if (_event_mask & EM_ALLOC) _alloc_engine->stop(); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 509603a1df..c4bafe9db8 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -90,9 +90,9 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; // Registration gate, set by openThreadRegistration() and cleared by - // closeThreadRegistration(). registerThread() is reached from the pthread - // hook without _state_lock, so it checks this flag to stay off the engines - // while start() sizes their per-thread structures (for example + // closeThreadRegistration() and ThreadTablesFreeze. registerThread() is + // reached from the pthread hook without _state_lock, so it checks this flag + // to stay off the engines while start() sizes their per-thread structures (for example // CTimer::resizeTimers()) and once stop() has torn them down. std::atomic _engines_ready; // Unregistration gate. Cleared by ThreadTablesFreeze for the part of start() @@ -106,14 +106,54 @@ class alignas(alignof(SpinLock)) Profiler { // the hook writes it on every native thread start and exit, while // isRunning() reads _state on every intercepted socket call. alignas(DEFAULT_CACHE_LINE_SIZE) std::atomic _hook_calls_inflight; - // Clears _thread_tables_stable (waiting for in-flight hook calls) for its - // lifetime; see profiler.cpp. - class ThreadTablesFreeze; + // Wait budget of waitForHookCalls(); HOOK_CALLS_DRAIN_TIMEOUT_NS (1 s) in + // profiler.cpp. Written only by the friend test accessor. long long: test + // budgets of several seconds must not overflow a 32-bit long. + static long long _hook_drain_timeout_ns; + + // Keeps Profiler::registerThread() and unregisterThread() off the engines + // while start() selects the CPU engine and lets it replace its per-thread + // table. Both gates are set again by openThreadRegistration() once the table + // is sized. Unregistration is reopened by the destructor at the latest, when + // start() returns; registration stays closed unless the CPU engine started + // (see Profiler::startCpuEngine()). An exit skipped in this window is + // cleaned up only if the thread is gone when the engine lists threads: + // CTimer::stop() deleted every timer, and PerfEvents::start() closes the fd + // of every listed-missing tid. A thread that is still listed while it exits + // is registered by start() and its slot stays occupied after it dies + // (PROF-16359). + class ThreadTablesFreeze { + public: + explicit ThreadTablesFreeze(Profiler *profiler) + : _profiler(profiler), _drained(false) { + // Both gates: registerThread() must not reach a table being replaced + // either, even if an earlier stop() left _engines_ready set. + _profiler->_engines_ready.store(false, std::memory_order_seq_cst); + _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); + _drained = _profiler->waitForHookCalls(); + } + ~ThreadTablesFreeze() { + _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); + } + // false if hook calls were still in flight after the wait budget; the + // caller must then fail without replacing any table. + bool drained() const { return _drained; } + ThreadTablesFreeze(const ThreadTablesFreeze &) = delete; + ThreadTablesFreeze &operator=(const ThreadTablesFreeze &) = delete; + + private: + Profiler *_profiler; + bool _drained; + }; // Closes the gate opened by openThreadRegistration() and waits for hook - // calls already past it to leave the engines. - void closeThreadRegistration(); - // Waits until no hook call is inside an engine. - void waitForHookCalls(); + // calls already past it to leave the engines. Returns false if calls are + // still in flight after the wait budget (logged); the caller must not tear + // down or replace any table then. + bool closeThreadRegistration(); + // Waits until no hook call is inside an engine, for at most + // _hook_drain_timeout_ns. Returns false on timeout (logged); the caller + // must not tear down or replace any table then. + bool waitForHookCalls(); // Gate checks and engine calls behind registerThread()/unregisterThread(), // run with thread cancellation disabled. static int registerThreadInEngines(int tid); @@ -196,6 +236,12 @@ class alignas(alignof(SpinLock)) Profiler { static bool prewarmUnwinder(); void disableEngines(); + // Starts _cpu_engine. On success opens thread registration (idempotent for + // engines that already opened it); on failure closes it again and logs the + // error. Returns whether the engine started. + bool startCpuEngine(Arguments &args); + // Enables the CPU engine's events iff EM_CPU is in activated. + void enableCpuEngine(int activated); void onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); void onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); @@ -410,11 +456,16 @@ class alignas(alignof(SpinLock)) Profiler { * the engines. A CPU engine with a per-thread table (CTimer, PerfEvents) * calls this from its start() once the table is sized and before it lists * the existing threads, so the hook handles a thread that starts or exits - * while start() registers the existing ones. For CTimer this misses no - * thread: the per-slot CAS resolves the hook and start() registering one - * tid. PerfEvents::start() still closes the fd of every tid missing from - * its thread list, including one the hook registered after the list was - * taken, so such a thread is not sampled (PROF-16359). Engines without + * while start() registers the existing ones. Profiler::startCpuEngine() + * calls it again once the CPU engine's start() succeeds, and closes the + * gate if it fails, so the gate is open iff the CPU engine started; + * wall-only, noop and failed-CPU recordings leave it closed. A future wall + * engine that needs registerThread() must open the gate itself. For CTimer + * this misses no thread: the per-slot CAS resolves the hook and start() + * registering one tid. PerfEvents::start()'s stale-fd sweep may close the + * fd of a tid the hook registered after the thread list was taken; a + * second listing pass re-registers it, so such a thread misses only the + * samples between the two passes (PROF-16359). Engines without * registerThread() never need the gate. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index e518175001..1c1e098b54 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -20,7 +20,9 @@ #include #include #include +#include #include +#include // The failure-path tests force calloc to fail with a negative count, which // makes count * size overflow. The CI sanitizer jobs run gtest binaries @@ -41,9 +43,35 @@ class ProfilerTestAccessor { static int hookCallsInflight(Profiler *p) { return p->_hook_calls_inflight.load(std::memory_order_acquire); } - static void closeThreadRegistration(Profiler *p) { - p->closeThreadRegistration(); + static bool closeThreadRegistration(Profiler *p) { + return p->closeThreadRegistration(); } + static bool startCpuEngine(Profiler *p, Arguments &args) { + return p->startCpuEngine(args); + } + static void enableCpuEngine(Profiler *p, int activated) { + p->enableCpuEngine(activated); + } + // Runs body(drained) while a real ThreadTablesFreeze is alive. + template static void withThreadTablesFreeze(Profiler *p, F body) { + Profiler::ThreadTablesFreeze freeze(p); + body(freeze.drained()); + } + + // Sets the hook-call wait budget for its lifetime and restores the previous + // one on destruction, including when a test assertion returns early. + class DrainBudget { + public: + explicit DrainBudget(long long ns) : _saved(Profiler::_hook_drain_timeout_ns) { + Profiler::_hook_drain_timeout_ns = ns; + } + ~DrainBudget() { Profiler::_hook_drain_timeout_ns = _saved; } + DrainBudget(const DrainBudget &) = delete; + DrainBudget &operator=(const DrainBudget &) = delete; + + private: + long long _saved; + }; static bool threadTablesStable(Profiler *p) { return p->_thread_tables_stable.load(std::memory_order_acquire); } @@ -131,6 +159,27 @@ class CTimerTestAccessor { CTimer::_timers = timers; CTimer::_max_timers = max_timers; } + + // Installs a caller-owned table for its lifetime and puts the previous one + // back on destruction, without freeing either. + class TimerTableSwap { + public: + TimerTableSwap(int *timers, int max_timers) + : _timers(CTimer::_timers), _max_timers(CTimer::_max_timers) { + CTimer::_timers = timers; + CTimer::_max_timers = max_timers; + } + ~TimerTableSwap() { + CTimer::_timers = _timers; + CTimer::_max_timers = _max_timers; + } + TimerTableSwap(const TimerTableSwap &) = delete; + TimerTableSwap &operator=(const TimerTableSwap &) = delete; + + private: + int *_timers; + int _max_timers; + }; }; // Returns the current SIGPROF handler address (sa_handler and sa_sigaction @@ -307,11 +356,16 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); ProfilerTestAccessor::RegistrationGate gate(profiler); + // Well above the 50 ms check below and below BlockingEngine's 5 s hold, so a + // slow sanitizer host does not turn the wait into a false timeout. + ProfilerTestAccessor::DrainBudget budget(4000000000LL); + std::thread hook([] { Profiler::registerThread(OS::threadId()); }); bool entered = waitFor([&] { return cpu.entered.load(); }); std::atomic closed{false}; + std::atomic closed_ok{false}; std::thread closer([&] { - ProfilerTestAccessor::closeThreadRegistration(profiler); + closed_ok.store(ProfilerTestAccessor::closeThreadRegistration(profiler)); closed.store(true); }); // Wait until the closer has cleared the gate, so it is in the wait loop. @@ -325,8 +379,280 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationWaitsForHookCalls) { hook.join(); closer.join(); EXPECT_TRUE(closed.load()); + EXPECT_TRUE(closed_ok.load()); + EXPECT_TRUE(entered); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// The wait is bounded: a hook call stuck inside an engine makes +// closeThreadRegistration() report failure after the budget instead of +// hanging stop() under _state_lock. The registration gate stays closed, and +// the unregistration gate is not touched. +TEST(ProfilerEnginesGateTest, CloseThreadRegistrationTimesOutOnStuckHookCall) { + Profiler *profiler = Profiler::instance(); + BlockingEngine cpu; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + ProfilerTestAccessor::RegistrationGate gate(profiler); + ProfilerTestAccessor::DrainBudget budget(200000000LL); + + std::thread hook([] { Profiler::registerThread(OS::threadId()); }); + bool entered = waitFor([&] { return cpu.entered.load(); }); + auto begin = std::chrono::steady_clock::now(); + bool ok = ProfilerTestAccessor::closeThreadRegistration(profiler); + long long elapsed_ms = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin) + .count(); + bool ready = ProfilerTestAccessor::enginesReady(profiler); + int inflight = ProfilerTestAccessor::hookCallsInflight(profiler); + bool stable = ProfilerTestAccessor::threadTablesStable(profiler); + + cpu.released.store(true); + hook.join(); + + EXPECT_TRUE(entered); + EXPECT_FALSE(ok); + EXPECT_FALSE(ready); + EXPECT_EQ(1, inflight); + EXPECT_TRUE(stable); + EXPECT_GE(elapsed_ms, 150); + EXPECT_LT(elapsed_ms, 3000); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// start()'s ThreadTablesFreeze closes both gates before it waits for hook +// calls already inside an engine, does not return while one is, keeps +// unregisterThread() off the engines while alive, and on destruction reopens +// unregistration but not registration (only a CPU engine that started opens +// that). +TEST(ProfilerEnginesGateTest, ThreadTablesFreezeClosesGatesAndWaitsForHookCalls) { + Profiler *profiler = Profiler::instance(); + BlockingEngine cpu; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + ProfilerTestAccessor::RegistrationGate gate(profiler); + ProfilerTestAccessor::DrainBudget budget(4000000000LL); + int tid = OS::threadId(); + + std::thread hook([] { Profiler::registerThread(OS::threadId()); }); + bool entered = waitFor([&] { return cpu.entered.load(); }); + std::atomic in_body{false}; + std::atomic drained_inside{false}; + std::atomic stable_inside{true}; + std::atomic ready_inside{true}; + std::atomic unreg_inside{0}; + std::thread freezer([&] { + ProfilerTestAccessor::withThreadTablesFreeze(profiler, [&](bool drained) { + drained_inside.store(drained); + stable_inside.store(ProfilerTestAccessor::threadTablesStable(profiler)); + ready_inside.store(ProfilerTestAccessor::enginesReady(profiler)); + Profiler::resetUnregisterObservableForTest(); + Profiler::unregisterThread(tid); + unreg_inside.store(Profiler::lastUnregisteredTidForTest()); + in_body.store(true); + }); + }); + bool cleared = + waitFor([&] { return !ProfilerTestAccessor::threadTablesStable(profiler); }); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + bool in_body_early = in_body.load(); + + cpu.released.store(true); + hook.join(); + freezer.join(); + + EXPECT_TRUE(entered); + EXPECT_TRUE(cleared); + EXPECT_FALSE(in_body_early); + EXPECT_TRUE(in_body.load()); + EXPECT_TRUE(drained_inside.load()); + EXPECT_FALSE(stable_inside.load()); + EXPECT_FALSE(ready_inside.load()); + EXPECT_EQ(-1, unreg_inside.load()); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + Profiler::resetUnregisterObservableForTest(); +} + +// A freeze whose wait times out reports it, so start() can fail without +// replacing any table, and its destructor still reopens unregistration. +TEST(ProfilerEnginesGateTest, ThreadTablesFreezeReportsUndrainedOnStuckHookCall) { + Profiler *profiler = Profiler::instance(); + BlockingEngine cpu; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + ProfilerTestAccessor::RegistrationGate gate(profiler); + ProfilerTestAccessor::DrainBudget budget(200000000LL); + + std::thread hook([] { Profiler::registerThread(OS::threadId()); }); + bool entered = waitFor([&] { return cpu.entered.load(); }); + bool body_ran = false; + bool drained_inside = true; + ProfilerTestAccessor::withThreadTablesFreeze(profiler, [&](bool drained) { + body_ran = true; + drained_inside = drained; + }); + bool stable_after = ProfilerTestAccessor::threadTablesStable(profiler); + bool ready_after = ProfilerTestAccessor::enginesReady(profiler); + + cpu.released.store(true); + hook.join(); + EXPECT_TRUE(entered); + EXPECT_TRUE(body_ran); + EXPECT_FALSE(drained_inside); + EXPECT_TRUE(stable_after); + EXPECT_FALSE(ready_after); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// A CPU engine fake: start() optionally opens the registration gate the way +// CTimer/PerfEvents do, and fails or succeeds. Counts enableEvents(true) and +// registerThread() calls. +class FakeCpuEngine : public Engine { +public: + FakeCpuEngine(bool fail, bool open_gate) : _fail(fail), _open_gate(open_gate) {} + std::atomic gate_open_in_start{false}; + std::atomic enable_true_calls{0}; + std::atomic register_calls{0}; + + Error start(Arguments &args) override { + if (_open_gate) { + Profiler::instance()->openThreadRegistration(); + gate_open_in_start.store( + ProfilerTestAccessor::enginesReady(Profiler::instance())); + } + return _fail ? Error("injected CPU engine start failure") : Error::OK; + } + void enableEvents(bool enabled) override { + if (enabled) { + enable_true_calls.fetch_add(1); + } + } + int registerThread(int tid) override { + register_calls.fetch_add(1); + return 0; + } + +private: + bool _fail; + bool _open_gate; +}; + +// A CPU engine that opened the gate and then failed leaves it closed, so the +// hook never reaches an engine that did not start. +TEST(ProfilerEnginesGateTest, FailedCpuEngineStartClosesThreadRegistration) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + FakeCpuEngine cpu(/*fail=*/true, /*open_gate=*/true); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + Arguments args; + + bool started = ProfilerTestAccessor::startCpuEngine(profiler, args); + bool ready = ProfilerTestAccessor::enginesReady(profiler); + int registered = Profiler::registerThread(OS::threadId()); + // Restores the closed gate if the call under test regressed. + ProfilerTestAccessor::closeThreadRegistration(profiler); + + EXPECT_FALSE(started); + EXPECT_TRUE(cpu.gate_open_in_start.load()); + EXPECT_FALSE(ready); + EXPECT_EQ(-1, registered); + EXPECT_EQ(0, cpu.register_calls.load()); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + +// A CPU engine without a per-thread table (ITimer-like) never opens the gate +// itself; startCpuEngine() opens it once the engine started, so the hook +// reaches it. +TEST(ProfilerEnginesGateTest, StartedCpuEngineOpensThreadRegistration) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + FakeCpuEngine cpu(/*fail=*/false, /*open_gate=*/false); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + Arguments args; + + bool started = ProfilerTestAccessor::startCpuEngine(profiler, args); + bool ready = ProfilerTestAccessor::enginesReady(profiler); + Profiler::registerThread(OS::threadId()); + int registered = cpu.register_calls.load(); + bool closed_ok = ProfilerTestAccessor::closeThreadRegistration(profiler); + + EXPECT_TRUE(started); + EXPECT_TRUE(ready); + EXPECT_EQ(1, registered); + EXPECT_TRUE(closed_ok); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// The CPU engine is enabled only if its own start() succeeded, even when +// another mode activated. +TEST(ProfilerEnginesGateTest, CpuEngineEnabledOnlyIfItStarted) { + Profiler *profiler = Profiler::instance(); + FakeCpuEngine cpu(/*fail=*/true, /*open_gate=*/false); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + Arguments args; + + bool started = ProfilerTestAccessor::startCpuEngine(profiler, args); + ProfilerTestAccessor::closeThreadRegistration(profiler); + int activated = EM_WALL | (started ? EM_CPU : 0); + ProfilerTestAccessor::enableCpuEngine(profiler, activated); + EXPECT_FALSE(started); + EXPECT_EQ(0, cpu.enable_true_calls.load()); + + ProfilerTestAccessor::enableCpuEngine(profiler, EM_CPU | EM_WALL); + EXPECT_EQ(1, cpu.enable_true_calls.load()); +} + +// The idle fast path in registerThread() only short-cuts a closed gate: with +// the gate open the call still reaches the engine, and the in-flight count is +// back at zero afterwards either way. The return value is not checked with +// the gate open: the plain wall engine reports -1, which is OR-ed in. +TEST(ProfilerEnginesGateTest, RegisterThreadFastPathDoesNotSkipOpenGate) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + FakeCpuEngine cpu(/*fail=*/false, /*open_gate=*/false); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + + EXPECT_EQ(-1, Profiler::registerThread(OS::threadId())); + EXPECT_EQ(0, cpu.register_calls.load()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + { + ProfilerTestAccessor::RegistrationGate gate(profiler); + Profiler::registerThread(OS::threadId()); + EXPECT_EQ(1, cpu.register_calls.load()); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); + } + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); +} + +// A negative tid must not index before CTimer's table. The page before the +// table is inaccessible, so an access through _timers[-1] faults. +TEST(CTimerTableTest, NegativeTidIsRejected) { + size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + char *base = (char *)mmap(nullptr, 2 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, (void *)base); + ASSERT_EQ(0, mprotect(base, page_size, PROT_NONE)); + int *table = (int *)(base + page_size); + { + CTimerTestAccessor::TimerTableSwap swap(table, 16); + CTimer engine; + // unregisterThread() first: registerThread() may fail earlier, in + // timer_create, so this is the discriminating call. + engine.unregisterThread(-1); + EXPECT_EQ(-1, engine.registerThread(-1)); + for (int i = 0; i < 16; i++) { + EXPECT_EQ(0, table[i]) << "slot " << i; + } + } + munmap(base, 2 * page_size); } // Thread body for the cancellation test: makes a hook call with asynchronous diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index 2dae58aa47..bf0d761c08 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -19,11 +19,26 @@ #ifdef __linux__ #include "arguments.h" +#include "nativeMem.h" #include "perfEvents.h" +#include #include #include +#include #include +#include + +// The failure-path tests force calloc to fail with a negative count, which +// makes count * size overflow. The CI sanitizer jobs run gtest binaries +// without *SAN_OPTIONS, so allocator_may_return_null defaults to 0 and the +// sanitizer reports that overflow as a fatal error instead of returning NULL. +#if defined(ASAN_ENABLED) || defined(TSAN_ENABLED) +#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() \ + GTEST_SKIP() << "sanitizer allocators abort on calloc overflow" +#else +#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() +#endif static int compare(int a, int b) { return PerfEvents::compareTids(&a, &b); } @@ -81,6 +96,34 @@ class PerfEventsTestAccessor { int _max_events; Ring _ring; }; + + static bool resizeEvents(PerfEvents &engine, int max_events) { + return engine.resizeEvents(max_events); + } + static PerfEvent *events() { return PerfEvents::_events; } + static int maxEvents() { return PerfEvents::_max_events; } + static PerfEvent *eventFor(int tid) { return PerfEvents::eventFor(tid); } + static unsigned int gen() { return PerfEvents::_events_gen; } + static void setGen(unsigned int gen) { PerfEvents::_events_gen = gen; } + static void setSlot(int tid, int fd, void *page) { + PerfEvents::setSlotForTest(tid, fd, page); + } + static int slotFd(int tid) { return PerfEvents::slotFdForTest(tid); } + + // Hands the current table to the caller without freeing it, so a test can + // start from an empty table and later reinstate the original one. + static void detach() { + PerfEvents::_events = nullptr; + PerfEvents::_max_events = -1; + } + + // Frees the test-owned table and reinstates the detached one, without + // touching the gauge (the fixture restores that separately). + static void reattach(PerfEvent *events, int max_events) { + free(PerfEvents::_events); + PerfEvents::_events = events; + PerfEvents::_max_events = max_events; + } }; // After a failed table allocation, start() leaves _events NULL with @@ -118,4 +161,204 @@ TEST(PerfEventsTableTest, SignalPathReadersRejectOutOfRangeTid) { munmap(page, 1 << 16); } +// Runs resizeEvents() against an empty table and an NM_PERF gauge starting at +// zero, and puts the original table, generation and gauge back afterwards. +class PerfEventsResizeTest : public ::testing::Test { +protected: + PerfEvent *_saved_events; + int _saved_max; + unsigned int _saved_gen; + long long _saved_live; + size_t _page_size; + + void SetUp() override { + _saved_events = PerfEventsTestAccessor::events(); + _saved_max = PerfEventsTestAccessor::maxEvents(); + _saved_gen = PerfEventsTestAccessor::gen(); + _saved_live = NativeMem::live(NM_PERF); + _page_size = (size_t)sysconf(_SC_PAGESIZE); + PerfEventsTestAccessor::detach(); + NativeMem::setLive(NM_PERF, 0); + } + + void TearDown() override { + PerfEventsTestAccessor::reattach(_saved_events, _saved_max); + PerfEventsTestAccessor::setGen(_saved_gen); + NativeMem::setLive(NM_PERF, _saved_live); + } + + // Puts a pipe read end and a 2-page anonymous mapping into slot tid, and + // accounts the ring the way registerThread() does. Returns the write end + // through *write_fd. + void armSlot(int tid, int *read_fd, int *write_fd, void **page) { + int fds[2]; + ASSERT_EQ(0, pipe(fds)); + *read_fd = fds[0]; + *write_fd = fds[1]; + *page = mmap(nullptr, 2 * _page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, *page); + PerfEventsTestAccessor::setSlot(tid, *read_fd, *page); + NativeMem::record(NM_PERF, (long long)(2 * _page_size)); + } +}; + +// A failed allocation leaves a NULL table with a -1 bound, not a NULL table +// with a stale bound, releases the old table's bytes from the gauge, and a +// later resize recovers. +TEST_F(PerfEventsResizeTest, FailedAllocationLeavesNoTable) { + SKIP_IF_CALLOC_OVERFLOW_ABORTS(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 1024)); + ASSERT_NE(nullptr, PerfEventsTestAccessor::events()); + EXPECT_EQ(1024, PerfEventsTestAccessor::maxEvents()); + long long one = NativeMem::live(NM_PERF); + EXPECT_GT(one, 0); + + EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); + EXPECT_EQ(nullptr, PerfEventsTestAccessor::events()); + EXPECT_EQ(-1, PerfEventsTestAccessor::maxEvents()); + EXPECT_EQ(0, NativeMem::live(NM_PERF)); + EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(0)); + // The failure state reads as "not started" and rejects every tid. + EXPECT_EQ(0, engine.registerThread(5)); + engine.unregisterThread(5); + + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 1024)); + EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(1023)); + EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(1024)); + EXPECT_EQ(one, NativeMem::live(NM_PERF)); +} + +// Replacing the table closes every fd and unmaps every ring the old table +// holds: once it is freed nothing could reach them. The fd/address checks +// run right after the resize, before anything else in the test opens or maps, +// so the fd number or address cannot have been reused (gtest is +// single-threaded here). +TEST_F(PerfEventsResizeTest, ResizeReleasesOldSlots) { + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + long long base = NativeMem::live(NM_PERF); + int fd = -1, write_fd = -1; + void *page = nullptr; + armSlot(5, &fd, &write_fd, &page); + ASSERT_FALSE(HasFatalFailure()); + ASSERT_EQ(base + (long long)(2 * _page_size), NativeMem::live(NM_PERF)); + + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 128)); + errno = 0; + int fcntl_rc = fcntl(fd, F_GETFD); + int fcntl_errno = errno; + errno = 0; + int msync_rc = msync(page, _page_size, MS_ASYNC); + int msync_errno = errno; + + EXPECT_EQ(-1, fcntl_rc); + EXPECT_EQ(EBADF, fcntl_errno); + EXPECT_EQ(-1, msync_rc); + EXPECT_EQ(ENOMEM, msync_errno); + EXPECT_EQ(2 * base, NativeMem::live(NM_PERF)); + EXPECT_EQ(0, PerfEventsTestAccessor::slotFd(5)); + close(write_fd); +} + +// The failure path releases the old table's slots too. +TEST_F(PerfEventsResizeTest, FailedResizeReleasesOldSlots) { + SKIP_IF_CALLOC_OVERFLOW_ABORTS(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + int fd = -1, write_fd = -1; + void *page = nullptr; + armSlot(5, &fd, &write_fd, &page); + ASSERT_FALSE(HasFatalFailure()); + + EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); + errno = 0; + int fcntl_rc = fcntl(fd, F_GETFD); + int fcntl_errno = errno; + errno = 0; + int msync_rc = msync(page, _page_size, MS_ASYNC); + int msync_errno = errno; + + EXPECT_EQ(-1, fcntl_rc); + EXPECT_EQ(EBADF, fcntl_errno); + EXPECT_EQ(-1, msync_rc); + EXPECT_EQ(ENOMEM, msync_errno); + EXPECT_EQ(0, NativeMem::live(NM_PERF)); + close(write_fd); +} + +// An unchanged size keeps the table, its fds and the gauge, so fds survive a +// stop()/start() when pid_max did not change. +TEST_F(PerfEventsResizeTest, SameSizeKeepsTable) { + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + int fds[2]; + ASSERT_EQ(0, pipe(fds)); + PerfEventsTestAccessor::setSlot(3, fds[0], nullptr); + PerfEvent *table = PerfEventsTestAccessor::events(); + unsigned int gen = PerfEventsTestAccessor::gen(); + long long live = NativeMem::live(NM_PERF); + + bool resized = PerfEventsTestAccessor::resizeEvents(engine, 64); + int fcntl_rc = fcntl(fds[0], F_GETFD); + + EXPECT_TRUE(resized); + EXPECT_EQ(table, PerfEventsTestAccessor::events()); + EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); + EXPECT_EQ(live, NativeMem::live(NM_PERF)); + EXPECT_NE(-1, fcntl_rc); + PerfEventsTestAccessor::setSlot(3, 0, nullptr); + close(fds[0]); + close(fds[1]); +} + +// eventFor() rejects the table while the generation is odd (a replacement in +// progress), and every resize, failed or not, advances the generation by two +// and leaves it even. +TEST_F(PerfEventsResizeTest, EventForRejectsOddGenerationAndResizeAdvancesIt) { + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + unsigned int g = PerfEventsTestAccessor::gen(); + EXPECT_EQ(0u, g & 1); + EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(3)); + + PerfEventsTestAccessor::setGen(g + 1); + EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(3)); + PerfEventsTestAccessor::setGen(g); + EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(3)); + + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 128)); + EXPECT_EQ(g + 2, PerfEventsTestAccessor::gen()); +} + +TEST_F(PerfEventsResizeTest, FailedResizeAdvancesGenerationToEven) { + SKIP_IF_CALLOC_OVERFLOW_ABORTS(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + unsigned int g = PerfEventsTestAccessor::gen(); + EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); + EXPECT_EQ(g + 2, PerfEventsTestAccessor::gen()); + EXPECT_EQ(0u, PerfEventsTestAccessor::gen() & 1); +} + +// A negative tid must not index before the table. The page before the table +// is inaccessible, so an access through _events[-1] faults. +TEST(PerfEventsTableTest, NegativeTidIsRejected) { + size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + char *base = (char *)mmap(nullptr, 3 * page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, (void *)base); + ASSERT_EQ(0, mprotect(base, page_size, PROT_NONE)); + { + PerfEventsTestAccessor::TableSwap swap( + reinterpret_cast(base + page_size), 4, RING_KERNEL); + PerfEvents engine; + EXPECT_EQ(-1, engine.registerThread(-1)); + engine.unregisterThread(-1); + EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(-1)); + } + munmap(base, 3 * page_size); +} + #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 9eb60cd6aa..3b5ed34697 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -66,15 +66,20 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ``` Profiler::start() profiler.cpp:1555 - ThreadTablesFreeze unregisterThread() gate closed until - the CPU engine sizes its table, at the - latest until return + ThreadTablesFreeze both gates closed until the CPU engine + sizes its table (openThreadRegistration()); + unregistration reopened at the latest on + return. Waits (bounded, 1 s) for hook + calls; on timeout start() fails with a + retryable error before selecting engines selectCpuEngine / selectWallEngine _jfr.start() SamplerPerf::primeClock() before any handler can fire - _cpu_engine->start() (EM_CPU) failure -> warning, continue - CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads + Profiler::startCpuEngine() (EM_CPU) failure -> warning, continue + _cpu_engine->start() + CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads on failure: closeThreadRegistration() + on success: openThreadRegistration() no-op for CTimer/PerfEvents _wall_engine->enableEvents(true) BEFORE start(): see below _wall_engine->start() (EM_WALL) _alloc_engine = selectAllocEngine(); start() (EM_ALLOC) @@ -82,13 +87,13 @@ Profiler::start() profiler.cpp:1555 NativeSocketSampler::start() (EM_NATIVESOCKET) if anything activated: switchThreadEvents(JVMTI_ENABLE) - _cpu_engine->enableEvents(true) LAST, only if EM_CPU activated + Profiler::enableCpuEngine() LAST, only if EM_CPU activated ``` - The wall engine is enabled before its thread starts because `timerLoopCommon()` checks `_enabled` once on entry and returns if it is - false (`wallClock.h:50`, `profiler.cpp:1795-1801`). -- The CPU engine is enabled last (`profiler.cpp:1901`). CPU handlers check + false (`wallClock.h:50`, the EM_WALL branch of `Profiler::start()`). +- The CPU engine is enabled last (`Profiler::enableCpuEngine()`). CPU handlers check `_enabled` on every signal, so nothing is lost by enabling late. It is not enabled when its own `start()` failed, even if another mode activated, so fds or timers left armed by an earlier recording cannot emit CPU samples. @@ -104,9 +109,18 @@ Profiler::start() profiler.cpp:1555 and tears nothing down (`profiler.cpp:1935`). 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook call that passed the check is still inside an engine. - No thread-hook call (pthread hook or JVMTI callback) can then arm a timer - after `CTimer::stop()`, or still be using the timer table when the next - `start()` resizes it. + No `registerThread()` call (pthread hook or JVMTI `ThreadStart`) can then arm + a timer or open an fd after the engines stop, or still be using a table when + the next `start()` resizes it. The wait is bounded (1 s). On timeout `stop()` + returns an error, tears nothing down and keeps the state `RUNNING`, like + step 2; a retried `stop()` runs every step again. + Unregistration is not closed here: JVMTI `ThreadEnd` and native thread exits + may still reach `unregisterThread()` while the engines stop. `CTimer::stop()` + and `CTimer::unregisterThread()` each release a slot with a per-slot CAS and + neither frees the table; `PerfEvents::stop()` is a no-op and keeps its fds, + which only `unregisterThread()` closes (under the fd CAS and the slot lock); + the wall engines and `ITimer` have no `unregisterThread()`. Only `start()` + replaces tables, under `ThreadTablesFreeze`. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. @@ -132,10 +146,19 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: and is installed before the engines start, so both calls are gated: - `registerThread()` needs `_engines_ready`, which `CTimer::start()` and `PerfEvents::start()` set via `Profiler::openThreadRegistration()` once - their table is sized and before they list threads. `stop()` clears it. - `PerfEvents::start()` still closes the fd of every tid missing from its - thread list, including one the hook registered after the list was taken - (PROF-16359). + their table is sized and before they list threads, and which + `Profiler::startCpuEngine()` sets once any CPU engine (including the + table-less `ITimer`, `ITimerJvmti` and J9 engines) has started. It is + cleared by `stop()`, by a failed CPU engine start and by + `ThreadTablesFreeze`, so the gate is open iff the CPU engine started. + Wall-only, noop and failed-CPU recordings leave it closed; no wall engine + implements `registerThread()`, and one that ever does must open the gate + itself. While the gate is closed, `registerThread()` returns before it + touches the in-flight counter. + - `PerfEvents::start()`'s stale-fd sweep may close the fd of a tid the hook + registered after the thread list was taken; a second listing pass then + re-registers every live thread, so such a thread misses only the samples + between the two passes (PROF-16359). - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only while `start()` may replace the table (`ThreadTablesFreeze`). It stays set while idle, because `PerfEvents::stop()` keeps per-thread fds open @@ -147,8 +170,10 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: Every `Profiler::registerThread()`/`unregisterThread()` call counts itself in flight (`HookCallScope`) for as long as it may be inside an engine. Closing a gate (`closeThreadRegistration()` in `stop()`, `ThreadTablesFreeze` in `start()`) waits -for that count to reach zero, so no call is inside an engine while it is torn down -or its table is replaced. The gate flags and the counter use `seq_cst` on both +for that count to reach zero, at most 1 s, so no `registerThread()` call is inside +an engine once `stop()` tears it down, and no call at all while `start()` replaces +its table. If the wait times out, the caller returns a retryable error and +neither tears down nor replaces anything. The gate flags and the counter use `seq_cst` on both sides: either a call sees the gate closed, or the closer sees the call and waits. Thread cancellation is disabled for the length of each call, because the engine @@ -242,7 +267,8 @@ resolved through `Arguments::cpuSamplerInterval()`. sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above - `_max_timers` logs a warning and is not profiled. + `_max_timers` logs a warning and is not profiled. The table is accounted + under `NM_CTIMER` at its full allocated size (see Profiler Self-Accounting). - **`stop()`** unregisters every slot (`ctimer_linux.cpp:212-218`). - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` @@ -271,7 +297,13 @@ resolved through `Arguments::cpuSamplerInterval()`. stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted - under `NM_PERF`. + under `NM_PERF`. The array is counted at its full allocated size, which is + mostly untouched address space (see Profiler Self-Accounting). +- **Table replacement**: only `resizeEvents()`, called from `start()` when + `pid_max` changed or after a failed allocation, replaces `_events`. It closes + every fd and unmaps every ring of the old table before freeing it, and + brackets the replacement with a seqlock generation so the signal-path + readers (`eventFor()`) never pair a table with another table's bound. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this engine (`profiler.cpp:1725`, `1745`). @@ -515,10 +547,25 @@ categories, defined in `nativeMem.h`: `NM_CALLTRACE`, `NM_DICTIONARY`, ..., exported as the `NATIVE_MEM_LIVE_BYTES`, `NATIVE_MEM_AVG_BYTES`, and `NATIVE_MEM_MAX_BYTES` counters (`flightRecorder.cpp:2201-2203`). -The engines above feed it: `PerfEvents` reports `NM_PERF`, `CTimer` reports -its tid-indexed timer table (`pid_max × sizeof(int)`) as `NM_CTIMER`, -`timerLoopCommon()` reports `NM_WALLCLOCK`, and `LivenessTracker` reports -`NM_LIVENESS`. +The engines above feed it: `PerfEvents` reports its tid-indexed event table +(up to `pid_max × sizeof(PerfEvent)` bytes, two cache lines per slot; address +space) and the per-thread ring pages (resident) as `NM_PERF`, `CTimer` reports +its tid-indexed timer table (up to `pid_max × 4` bytes, `pid_max × sizeof(int)`; +address space) as `NM_CTIMER`, `timerLoopCommon()` reports `NM_WALLCLOCK`, and +`LivenessTracker` reports `NM_LIVENESS`. + +Unlike the other categories, the two tid-indexed tables are allocated address +space, not resident memory: each is `calloc`ed at `pid_max` slots, but only the +pages holding live tids are ever touched. With a `pid_max` of 4194304 the +`ctimer` gauge reads about 16 MiB and the table part of the `perf` gauge about +512 MiB, while the resident share is usually far lower. Read both as upper +bounds. Only `NM_PERF`'s ring pages are resident. + +Each category is exported per chunk as +`native_mem_{live,avg,max,chunk_overhead}_bytes.`, with names taken +from `DD_NATIVE_MEM_CATEGORY_TABLE`. The set grows when categories are added +(`ctimer` is the newest), so consumers should match these counters by prefix +rather than hard-code the category list. ## Counters by Engine diff --git a/doc/reference/ProfilerMemoryRequirements.md b/doc/reference/ProfilerMemoryRequirements.md index 03ddb57510..81ee6eec68 100644 --- a/doc/reference/ProfilerMemoryRequirements.md +++ b/doc/reference/ProfilerMemoryRequirements.md @@ -7,11 +7,14 @@ The Datadog Java Profiler has inherent memory requirements due to its architectu The sizes below are estimates. The profiler's own native allocations are measured at runtime: `NativeMem` (`ddprof-lib/src/main/cpp/nativeMem.h`) keeps live, average and peak bytes for each category (`calltrace`, `dictionary`, `line_tables`, `thread_local`, -`jfr_buffers`, `method_map`, ...), and every chunk carries them as +`jfr_buffers`, `method_map`, `perf`, `ctimer`, ...), and every chunk carries them as `datadog.ProfilerCounter` events named `native_mem_live_bytes.`, `native_mem_avg_bytes.`, `native_mem_max_bytes.` and `native_mem_chunk_overhead_bytes.`. jmethodIDs are allocated by the JVM and are -not part of these figures. +not part of these figures. The category set is not fixed (`ctimer` is the newest), so +match these counters by prefix rather than by a fixed list. The `ctimer` gauge and the +table part of the `perf` gauge count reserved, mostly untouched tid-indexed tables +(`pid_max` slots each), so they overstate RSS and are upper bounds. ## Memory Requirements From f95195e1b58bd5bad3dc98569b89796a6743d47f Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 29 Oct 2026 14:26:58 -0400 Subject: [PATCH 16/26] PROF-16213: Fix ctimer_ut compilation on Linux NegativeTidIsRejected called CTimer::registerThread() and unregisterThread() directly, but both are protected. Route the calls through CTimerTestAccessor, which CTimer befriends. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 1c1e098b54..5cc5aaeb1e 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -144,6 +144,12 @@ class CTimerTestAccessor { int max_timers) { return engine.startWithTableSize(args, max_timers); } + static int registerThread(CTimer &engine, int tid) { + return engine.registerThread(tid); + } + static void unregisterThread(CTimer &engine, int tid) { + engine.unregisterThread(tid); + } // Hands the current table to the caller without freeing it, so a test can // run against an empty table and later reinstate the original one. @@ -646,8 +652,8 @@ TEST(CTimerTableTest, NegativeTidIsRejected) { CTimer engine; // unregisterThread() first: registerThread() may fail earlier, in // timer_create, so this is the discriminating call. - engine.unregisterThread(-1); - EXPECT_EQ(-1, engine.registerThread(-1)); + CTimerTestAccessor::unregisterThread(engine, -1); + EXPECT_EQ(-1, CTimerTestAccessor::registerThread(engine, -1)); for (int i = 0; i < 16; i++) { EXPECT_EQ(0, table[i]) << "slot " << i; } From 404fc14a8260f938124b38f217ec0fb46e3e5e77 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 29 Oct 2026 15:33:51 -0400 Subject: [PATCH 17/26] PROF-16213: Fix ResizeReleasesOldSlots on musl musl's malloc maps the new table with mmap, and the kernel can place it at the address the unmapped ring just released, so msync() on the old ring address succeeds. Check the ring through a sentinel read back via a pipe instead: a fresh mapping does not carry the sentinel, and the pipe read fails with EFAULT rather than faulting on an unmapped or PROT_NONE page. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 46 ++++++++++++++++------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index bf0d761c08..e1aac56fdc 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -198,9 +199,32 @@ class PerfEventsResizeTest : public ::testing::Test { *page = mmap(nullptr, 2 * _page_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); ASSERT_NE(MAP_FAILED, *page); + *(uint64_t *)*page = RING_SENTINEL; PerfEventsTestAccessor::setSlot(tid, *read_fd, *page); NativeMem::record(NM_PERF, (long long)(2 * _page_size)); } + + // Reports whether the ring armSlot() mapped at page is still mapped. Once + // the ring is unmapped, the allocator may map new memory at the same + // address: musl's malloc does when resizeEvents() allocates the new table. + // So the check reads back the sentinel armSlot() wrote instead of only + // probing the address. The read goes through a pipe, which fails with + // EFAULT instead of faulting when the page is unmapped or inaccessible. + static bool ringMapped(void *page) { + int fds[2]; + if (pipe(fds) != 0) { + ADD_FAILURE() << "pipe() failed: " << errno; + return true; + } + uint64_t value = 0; + bool readable = write(fds[1], page, sizeof(value)) == sizeof(value) && + read(fds[0], &value, sizeof(value)) == sizeof(value); + close(fds[0]); + close(fds[1]); + return readable && value == RING_SENTINEL; + } + + static constexpr uint64_t RING_SENTINEL = 0x5a5a5a5a5a5a5a5aULL; }; // A failed allocation leaves a NULL table with a -1 bound, not a NULL table @@ -231,10 +255,10 @@ TEST_F(PerfEventsResizeTest, FailedAllocationLeavesNoTable) { } // Replacing the table closes every fd and unmaps every ring the old table -// holds: once it is freed nothing could reach them. The fd/address checks -// run right after the resize, before anything else in the test opens or maps, -// so the fd number or address cannot have been reused (gtest is -// single-threaded here). +// holds: once it is freed nothing could reach them. The fd check runs right +// after the resize, before anything else in the test opens a file, so the fd +// number cannot have been reused (gtest is single-threaded here). The ring +// address can be: see ringMapped(). TEST_F(PerfEventsResizeTest, ResizeReleasesOldSlots) { PerfEvents engine; ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); @@ -249,14 +273,11 @@ TEST_F(PerfEventsResizeTest, ResizeReleasesOldSlots) { errno = 0; int fcntl_rc = fcntl(fd, F_GETFD); int fcntl_errno = errno; - errno = 0; - int msync_rc = msync(page, _page_size, MS_ASYNC); - int msync_errno = errno; + bool mapped = ringMapped(page); EXPECT_EQ(-1, fcntl_rc); EXPECT_EQ(EBADF, fcntl_errno); - EXPECT_EQ(-1, msync_rc); - EXPECT_EQ(ENOMEM, msync_errno); + EXPECT_FALSE(mapped); EXPECT_EQ(2 * base, NativeMem::live(NM_PERF)); EXPECT_EQ(0, PerfEventsTestAccessor::slotFd(5)); close(write_fd); @@ -276,14 +297,11 @@ TEST_F(PerfEventsResizeTest, FailedResizeReleasesOldSlots) { errno = 0; int fcntl_rc = fcntl(fd, F_GETFD); int fcntl_errno = errno; - errno = 0; - int msync_rc = msync(page, _page_size, MS_ASYNC); - int msync_errno = errno; + bool mapped = ringMapped(page); EXPECT_EQ(-1, fcntl_rc); EXPECT_EQ(EBADF, fcntl_errno); - EXPECT_EQ(-1, msync_rc); - EXPECT_EQ(ENOMEM, msync_errno); + EXPECT_FALSE(mapped); EXPECT_EQ(0, NativeMem::live(NM_PERF)); close(write_fd); } From fd7482ff500561ec199cca5e53b69324eddf50d9 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 29 Oct 2026 15:56:24 -0400 Subject: [PATCH 18/26] PROF-16213: Skip calloc-overflow tests in perfEvents_ut under sanitizers SKIP_IF_CALLOC_OVERFLOW_ABORTS() tests ASAN_ENABLED/TSAN_ENABLED, which common.h defines, but perfEvents_ut.cpp did not include it. The macro expanded to nothing, so the allocation-failure tests called calloc(-1) under the ASan/TSan allocators, which abort the whole gtest binary. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index e1aac56fdc..e5e5502c6d 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -19,6 +19,7 @@ #ifdef __linux__ #include "arguments.h" +#include "common.h" #include "nativeMem.h" #include "perfEvents.h" From 2de3fa9f479b648acd7877b011a800ab91e45616 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 29 Oct 2026 17:05:04 -0400 Subject: [PATCH 19/26] PROF-16213: Freeze thread unregistration only around a table replacement Profiler::start() held ThreadTablesFreeze from CPU engine selection until the engine sized its table, a window that includes symbol loading and JFR start. A thread exit in that window was dropped. If the thread was still listed when the engine listed threads, its timer or fd stayed in the slot after it died, and a thread that later reused the tid could not claim it. start()/stop() cycles repeat the window. PerfEvents keeps its fds across stop(), so such a slot can outlive every later restart. On a restart with an unchanged pid_max no table is replaced, so the freeze protected nothing. Move it into new CTimer::sizeTimerTable() and PerfEvents::sizeEventTable(), which freeze only when the table size changes. On a drain timeout the engine's start() fails retryably and keeps its table. start() itself now only closes the registration gate. start() and check() replace _cpu_engine/_wall_engine while unregistration is open, so store them with release and load them with acquire in the hook paths. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 + ddprof-lib/src/main/cpp/ctimer.h | 3 + ddprof-lib/src/main/cpp/ctimer_linux.cpp | 31 ++++- ddprof-lib/src/main/cpp/perfEvents.h | 7 +- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 29 ++++- ddprof-lib/src/main/cpp/profiler.cpp | 43 ++++--- ddprof-lib/src/main/cpp/profiler.h | 79 ++++++------ ddprof-lib/src/test/cpp/ctimer_ut.cpp | 84 +++++++++++- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 127 +++++++++++++++++++ doc/architecture/ProfilingEngines.md | 37 ++++-- 10 files changed, 359 insertions(+), 84 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 76af8d151c..38e66f61ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -289,6 +289,9 @@ for a temporary window. registration gate once any CPU engine started, so the gate is open iff it did; - an engine's `stop()` must tolerate a concurrent `unregisterThread()`: `stop()` closes only the registration gate, and unregistration stays open while the engines stop; +- close unregistration (`ThreadTablesFreeze`) only around an actual table replacement, + never for a whole `start()`: an exit dropped while it is closed leaves a stale slot + that a thread reusing the tid cannot claim, and `start()`/`stop()` cycles repeat; - restore the cancellation state outside every destructor-bearing scope, never from a destructor: current glibc acts on a pending asynchronous cancellation as soon as it is re-enabled, and unwinding out of an implicitly `noexcept` destructor terminates. diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h index 3aac58372d..2b7fcce2da 100644 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ b/ddprof-lib/src/main/cpp/ctimer.h @@ -45,6 +45,9 @@ class CTimer : public Engine { static int *_timers; static bool resizeTimers(int max_timers); + // Sizes the timer table for start(), replacing it under a + // Profiler::ThreadTablesFreeze only when the size changes. + static Error sizeTimerTable(int max_timers); // start() with the timer table size passed in rather than read from // pid_max, so a test can drive the allocation-failure path. Error startWithTableSize(Arguments &args, int max_timers); diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index d674d844a1..886d62284f 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -132,10 +132,10 @@ void CTimer::unregisterThread(int tid) { } } -// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Runs before -// start() opens thread registration, so no thread-hook call (pthread hook or -// JVMTI ThreadStart/ThreadEnd, which share the gate) can be inside the table -// while it is replaced. +// Sizes the tid-indexed timer table and accounts it under NM_CTIMER. Called +// from sizeTimerTable() under a Profiler::ThreadTablesFreeze, so no +// thread-hook call (pthread hook or JVMTI ThreadStart/ThreadEnd) can be inside +// the table while it is replaced. // // On allocation failure the table is left empty (_max_timers == 0), so the tid // bound checks reject every tid instead of indexing a NULL table, and the next @@ -157,6 +157,24 @@ bool CTimer::resizeTimers(int max_timers) { return true; } +// A restart with an unchanged pid_max keeps the table and never closes thread +// unregistration, so an exit during start() still releases its timer. Only an +// actual replacement freezes the hook calls; if one is still inside the table +// after the wait budget, start() fails retryably and keeps the old table. +Error CTimer::sizeTimerTable(int max_timers) { + if (max_timers == _max_timers) { + return Error::OK; + } + Profiler::ThreadTablesFreeze freeze(Profiler::instance()); + if (!freeze.drained()) { + return Error("thread hook calls did not drain; retry start()"); + } + if (!resizeTimers(max_timers)) { + return Error("Failed to allocate CPU timer table"); + } + return Error::OK; +} + Error CTimer::check(Arguments &args) { timer_t timer; if (timer_create(CLOCK_THREAD_CPUTIME_ID, NULL, &timer) < 0) { @@ -180,8 +198,9 @@ Error CTimer::startWithTableSize(Arguments &args, int max_timers) { _cstack = args._cstack; _signal = SIGPROF; - if (!resizeTimers(max_timers)) { - return Error("Failed to allocate CPU timer table"); + Error error = sizeTimerTable(max_timers); + if (error) { + return error; } // Prime the origin-check cache from this non-signal context before any diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h index cd77eeb5dd..d37329d3e8 100644 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ b/ddprof-lib/src/main/cpp/perfEvents.h @@ -60,9 +60,12 @@ class PerfEvents : public Engine { // Replaces the tid-indexed table with one of max_events slots, after // releasing every fd and ring held by the old table. No-op when the size is // unchanged. On allocation failure leaves _events == NULL and - // _max_events == -1 and returns false. Called only from start(), with the - // Profiler's registration gates closed. + // _max_events == -1 and returns false. Called only from sizeEventTable(), + // with the Profiler's registration gates closed. bool resizeEvents(int max_events); + // Sizes the table for start(), replacing it under a + // Profiler::ThreadTablesFreeze only when the size changes. + Error sizeEventTable(int max_events); #ifdef UNIT_TEST // Stores fd and ring page into slot tid of the current table (test only). diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index b815166409..23e5a630c5 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -926,9 +926,9 @@ void PerfEvents::endTableUpdate() { __ATOMIC_RELEASE); } -// The only writer of _events and _max_events outside tests. start() runs under -// the Profiler's ThreadTablesFreeze with the registration gate closed, so the -// hook stays off the table; only the signal-path readers (eventFor()) run +// The only writer of _events and _max_events outside tests. sizeEventTable() +// runs it under the Profiler's ThreadTablesFreeze, so the hook stays off the +// table; only the signal-path readers (eventFor()) run // concurrently, and the generation bracket lets them reject a pair read while // the table is replaced. bool PerfEvents::resizeEvents(int max_events) { @@ -975,6 +975,24 @@ bool PerfEvents::resizeEvents(int max_events) { return events != NULL; } +// A restart with an unchanged pid_max keeps the table and never closes thread +// unregistration, so an exit during start() still closes its fd. Only an +// actual replacement freezes the hook calls; if one is still inside the table +// after the wait budget, start() fails retryably and keeps the old table. +Error PerfEvents::sizeEventTable(int max_events) { + if (max_events == _max_events) { + return Error::OK; + } + Profiler::ThreadTablesFreeze freeze(Profiler::instance()); + if (!freeze.drained()) { + return Error("thread hook calls did not drain; retry start()"); + } + if (!resizeEvents(max_events)) { + return Error("Failed to allocate perf event table"); + } + return Error::OK; +} + #ifdef UNIT_TEST void PerfEvents::setSlotForTest(int tid, int fd, void *page) { _events[tid]._fd = fd; @@ -1031,8 +1049,9 @@ Error PerfEvents::start(Arguments &args) { // mappings. resizeEvents() closes the fds of the old table before freeing // it; on allocation failure the table stays empty and the next start() // retries. - if (!resizeEvents(OS::getMaxThreadId())) { - return Error("Failed to allocate perf event table"); + Error error = sizeEventTable(OS::getMaxThreadId()); + if (error) { + return error; } OS::installSignalHandler(SIGPROF, signalHandler); diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 540d6b3100..de2d339ca4 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -285,8 +285,9 @@ int Profiler::registerThreadInEngines(int tid) { if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return -1; // not registered, as Engine::registerThread() reports it } - return _instance->_cpu_engine->registerThread(tid) | - _instance->_wall_engine->registerThread(tid); + Engine *cpu = __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE); + Engine *wall = __atomic_load_n(&_instance->_wall_engine, __ATOMIC_ACQUIRE); + return cpu->registerThread(tid) | wall->registerThread(tid); } #ifdef UNIT_TEST static std::atomic g_test_last_unregistered_tid{-1}; @@ -317,8 +318,14 @@ void Profiler::unregisterThreadInEngines(int tid) { g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); return; #endif - _instance->_cpu_engine->unregisterThread(tid); - _instance->_wall_engine->unregisterThread(tid); + // start() and check() may replace the engines concurrently. Either engine + // is safe to call: unregisterThread() only releases the exiting thread's own + // slot, bound-checked against the engine's current table, and a table being + // replaced is behind the gate above. + __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE) + ->unregisterThread(tid); + __atomic_load_n(&_instance->_wall_engine, __ATOMIC_ACQUIRE) + ->unregisterThread(tid); } const char *Profiler::asgctError(int code) { @@ -1146,8 +1153,8 @@ bool Profiler::startCpuEngine(Arguments &args) { Error error = _cpu_engine->start(args); if (error) { // The engine may have opened the gate before failing. A call that is - // still in flight after the wait budget is waited for again by the next - // start()'s ThreadTablesFreeze before any table is replaced. + // still in flight after the wait budget is waited for again by the + // ThreadTablesFreeze in front of the next table replacement. (void)closeThreadRegistration(); Log::warn("%s", error.message()); return false; @@ -1764,15 +1771,15 @@ Error Profiler::start(Arguments &args, bool reset) { _thread_filter.remove(slot_id); // Remove from filtering initially (matches onThreadStart behavior) } - ThreadTablesFreeze tables_freeze(this); - if (!tables_freeze.drained()) { - // Nothing selected or replaced yet; the freeze destructor reopens - // unregistration and _engines_ready stays closed. A later start() waits - // again. Retryable, like the earlier error returns in this function. - return Error("thread hook calls did not drain; retry start()"); - } - _cpu_engine = selectCpuEngine(args); - _wall_engine = selectWallEngine(args); + // Registration stays closed until the CPU engine started (see + // startCpuEngine()), even if an earlier stop() left it open. Unregistration + // stays open: a thread exit dropped while the profiler is alive leaves a + // stale slot, so only the table replacement inside the CPU engine's start() + // closes it (ThreadTablesFreeze). No wait here: nothing below touches a + // per-thread table before that freeze waits for the in-flight calls. + _engines_ready.store(false, std::memory_order_seq_cst); + __atomic_store_n(&_cpu_engine, selectCpuEngine(args), __ATOMIC_RELEASE); + __atomic_store_n(&_wall_engine, selectWallEngine(args), __ATOMIC_RELEASE); _cstack = args._cstack; _force_jmethodID = args._force_jmethodID; if (_cstack == CSTACK_DEFAULT) { @@ -2004,7 +2011,7 @@ Error Profiler::stop() { // CTimer::unregisterThread() each release a slot with a per-slot CAS and // neither frees the table; PerfEvents::stop() is a no-op and the slot is // released under its fd CAS and slot lock; wall engines and ITimer have no - // unregisterThread(). Only start() replaces tables, under + // unregisterThread(). Only a CPU engine's start() replaces its table, under // ThreadTablesFreeze. Unregistration stays open so PerfEvents can close the // fd of a thread that exits after stop(). if (!closeThreadRegistration()) { @@ -2147,11 +2154,11 @@ Error Profiler::check(Arguments &args) { error = checkJvmCapabilities(); if (!error && (args._event != NULL || args._cpu >= 0)) { - _cpu_engine = selectCpuEngine(args); + __atomic_store_n(&_cpu_engine, selectCpuEngine(args), __ATOMIC_RELEASE); error = _cpu_engine->check(args); } if (!error && args._wall >= 0) { - _wall_engine = selectWallEngine(args); + __atomic_store_n(&_wall_engine, selectWallEngine(args), __ATOMIC_RELEASE); error = _wall_engine->check(args); } if (!error && args._memory >= 0) { diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index c4bafe9db8..afa8ee3293 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -90,14 +90,14 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; // Registration gate, set by openThreadRegistration() and cleared by - // closeThreadRegistration() and ThreadTablesFreeze. registerThread() is + // start(), closeThreadRegistration() and ThreadTablesFreeze. registerThread() is // reached from the pthread hook without _state_lock, so it checks this flag // to stay off the engines while start() sizes their per-thread structures (for example // CTimer::resizeTimers()) and once stop() has torn them down. std::atomic _engines_ready; - // Unregistration gate. Cleared by ThreadTablesFreeze for the part of start() - // that may replace the engines' per-thread tables, and set again once they - // are sized. Unlike _engines_ready it stays set while the profiler is idle: + // Unregistration gate. Cleared by ThreadTablesFreeze only while a CPU + // engine replaces its per-thread table, and set again once the table is + // sized. Unlike _engines_ready it stays set while the profiler is idle: // PerfEvents keeps its per-thread fds across stop(), and only // unregisterThread() closes the fd of a thread that exits after stop(). std::atomic _thread_tables_stable; @@ -111,40 +111,6 @@ class alignas(alignof(SpinLock)) Profiler { // budgets of several seconds must not overflow a 32-bit long. static long long _hook_drain_timeout_ns; - // Keeps Profiler::registerThread() and unregisterThread() off the engines - // while start() selects the CPU engine and lets it replace its per-thread - // table. Both gates are set again by openThreadRegistration() once the table - // is sized. Unregistration is reopened by the destructor at the latest, when - // start() returns; registration stays closed unless the CPU engine started - // (see Profiler::startCpuEngine()). An exit skipped in this window is - // cleaned up only if the thread is gone when the engine lists threads: - // CTimer::stop() deleted every timer, and PerfEvents::start() closes the fd - // of every listed-missing tid. A thread that is still listed while it exits - // is registered by start() and its slot stays occupied after it dies - // (PROF-16359). - class ThreadTablesFreeze { - public: - explicit ThreadTablesFreeze(Profiler *profiler) - : _profiler(profiler), _drained(false) { - // Both gates: registerThread() must not reach a table being replaced - // either, even if an earlier stop() left _engines_ready set. - _profiler->_engines_ready.store(false, std::memory_order_seq_cst); - _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); - _drained = _profiler->waitForHookCalls(); - } - ~ThreadTablesFreeze() { - _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); - } - // false if hook calls were still in flight after the wait budget; the - // caller must then fail without replacing any table. - bool drained() const { return _drained; } - ThreadTablesFreeze(const ThreadTablesFreeze &) = delete; - ThreadTablesFreeze &operator=(const ThreadTablesFreeze &) = delete; - - private: - Profiler *_profiler; - bool _drained; - }; // Closes the gate opened by openThreadRegistration() and waits for hook // calls already past it to leave the engines. Returns false if calls are // still in flight after the wait budget (logged); the caller must not tear @@ -184,6 +150,9 @@ class alignas(alignof(SpinLock)) Profiler { ThreadFilter _thread_filter; CallTraceStorage _call_trace_storage; FlightRecorder _jfr; + // Replaced by start() and check() while thread unregistration is open, so + // the hook paths (registerThreadInEngines(), unregisterThreadInEngines()) + // load them with acquire and those writers store them with release. Engine *_cpu_engine; Engine *_wall_engine = NULL; Engine *_alloc_engine; @@ -323,6 +292,40 @@ class alignas(alignof(SpinLock)) Profiler { } public: + // Keeps Profiler::registerThread() and unregisterThread() off the engines + // while a CPU engine replaces its per-thread table. CTimer and PerfEvents + // construct it in start() only when the table size changes (pid_max changed, + // or no table yet): a thread exit skipped while it is alive is dropped, so it + // must not span a restart that keeps the table. Both gates are set again by + // openThreadRegistration() once the table is sized; the destructor reopens + // unregistration at the latest. An exit skipped in this window is cleaned up + // only if the thread is gone when the engine lists threads; a thread that is + // still listed while it exits is registered by start() and its slot stays + // occupied after it dies (PROF-16359). + class ThreadTablesFreeze { + public: + explicit ThreadTablesFreeze(Profiler *profiler) + : _profiler(profiler), _drained(false) { + // Both gates: registerThread() must not reach a table being replaced + // either, even if an earlier stop() left _engines_ready set. + _profiler->_engines_ready.store(false, std::memory_order_seq_cst); + _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); + _drained = _profiler->waitForHookCalls(); + } + ~ThreadTablesFreeze() { + _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); + } + // false if hook calls were still in flight after the wait budget; the + // caller must then fail without replacing any table. + bool drained() const { return _drained; } + ThreadTablesFreeze(const ThreadTablesFreeze &) = delete; + ThreadTablesFreeze &operator=(const ThreadTablesFreeze &) = delete; + + private: + Profiler *_profiler; + bool _drained; + }; + Profiler() : _state_lock(), _state(State::NEW), _engines_ready(false), _thread_tables_stable(true), _hook_calls_inflight(0), diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 5cc5aaeb1e..8edb635d22 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -72,6 +72,22 @@ class ProfilerTestAccessor { private: long long _saved; }; + // Counts a hook call in flight for its lifetime, as if one were stuck inside + // an engine, without needing a thread or an engine to hold it. + class HookCallHold { + public: + explicit HookCallHold(Profiler *p) : _p(p) { + p->_hook_calls_inflight.fetch_add(1, std::memory_order_seq_cst); + } + ~HookCallHold() { + _p->_hook_calls_inflight.fetch_sub(1, std::memory_order_release); + } + HookCallHold(const HookCallHold &) = delete; + HookCallHold &operator=(const HookCallHold &) = delete; + + private: + Profiler *_p; + }; static bool threadTablesStable(Profiler *p) { return p->_thread_tables_stable.load(std::memory_order_acquire); } @@ -138,6 +154,9 @@ template static bool waitFor(Cond cond) { class CTimerTestAccessor { public: static bool resizeTimers(int max_timers) { return CTimer::resizeTimers(max_timers); } + static Error sizeTimerTable(int max_timers) { + return CTimer::sizeTimerTable(max_timers); + } static int maxTimers() { return CTimer::_max_timers; } static int *timers() { return CTimer::_timers; } static Error startWithTableSize(CTimer &engine, Arguments &args, @@ -313,6 +332,68 @@ TEST_F(CTimerNativeMemTest, StartFailsWhenTableAllocationFails) { EXPECT_EQ(0, NativeMem::live(NM_CTIMER)); } +// A restart with an unchanged pid_max keeps the table without freezing thread +// unregistration, so a thread that exits during start() still releases its +// timer. A hook call is held in flight: a freeze would wait for it and time +// out, so success shows that none ran. +TEST_F(CTimerNativeMemTest, SameSizeRestartDoesNotFreezeUnregistration) { + Profiler *profiler = Profiler::instance(); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(64)); + int *table = CTimerTestAccessor::timers(); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + Error error = Error::OK; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + error = CTimerTestAccessor::sizeTimerTable(64); + } + + EXPECT_FALSE((bool)error); + EXPECT_EQ(table, CTimerTestAccessor::timers()); + EXPECT_EQ(64, CTimerTestAccessor::maxTimers()); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + +// A size change replaces the table only under a freeze. While a hook call is +// still in flight after the wait budget, start() fails retryably and keeps the +// old table, and unregistration is open again afterwards. +TEST_F(CTimerNativeMemTest, ResizeKeepsTableWhenHookCallsDoNotDrain) { + Profiler *profiler = Profiler::instance(); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(64)); + int *table = CTimerTestAccessor::timers(); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + Error error = Error::OK; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + error = CTimerTestAccessor::sizeTimerTable(128); + } + + EXPECT_TRUE((bool)error); + EXPECT_STREQ("thread hook calls did not drain; retry start()", + error.message()); + EXPECT_EQ(table, CTimerTestAccessor::timers()); + EXPECT_EQ(64, CTimerTestAccessor::maxTimers()); + EXPECT_EQ(bytes(64), NativeMem::live(NM_CTIMER)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); +} + +// With no hook call in flight, a size change replaces the table and reopens +// unregistration. +TEST_F(CTimerNativeMemTest, ResizeReplacesTableWhenHookCallsDrain) { + Profiler *profiler = Profiler::instance(); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(64)); + + Error error = CTimerTestAccessor::sizeTimerTable(128); + + EXPECT_FALSE((bool)error); + EXPECT_NE(nullptr, CTimerTestAccessor::timers()); + EXPECT_EQ(128, CTimerTestAccessor::maxTimers()); + EXPECT_EQ(bytes(128), NativeMem::live(NM_CTIMER)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + // Profiler::registerThread() is reached from the pthread hook without // _state_lock. Until start() has sized the engines' per-thread structures // (CTimer::resizeTimers()) and set the engines-ready flag, it must return @@ -426,7 +507,8 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationTimesOutOnStuckHookCall) { EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } -// start()'s ThreadTablesFreeze closes both gates before it waits for hook +// The ThreadTablesFreeze a CPU engine's start() takes around a table +// replacement closes both gates before it waits for hook // calls already inside an engine, does not return while one is, keeps // unregisterThread() off the engines while alive, and on destruction reopens // unregistration but not registration (only a CPU engine that started opens diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index e5e5502c6d..e362c361df 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -22,6 +22,7 @@ #include "common.h" #include "nativeMem.h" #include "perfEvents.h" +#include "profiler.h" #include #include @@ -44,6 +45,51 @@ static int compare(int a, int b) { return PerfEvents::compareTids(&a, &b); } +// The parts of the Profiler's thread-hook gates the table-sizing tests need. +// This binary defines its own copy of the friend accessor; ctimer_ut.cpp has +// the full one. +class ProfilerTestAccessor { +public: + static bool enginesReady(Profiler *p) { + return p->_engines_ready.load(std::memory_order_acquire); + } + static bool threadTablesStable(Profiler *p) { + return p->_thread_tables_stable.load(std::memory_order_acquire); + } + + // Sets the hook-call wait budget for its lifetime and restores the previous + // one on destruction, including when a test assertion returns early. + class DrainBudget { + public: + explicit DrainBudget(long long ns) : _saved(Profiler::_hook_drain_timeout_ns) { + Profiler::_hook_drain_timeout_ns = ns; + } + ~DrainBudget() { Profiler::_hook_drain_timeout_ns = _saved; } + DrainBudget(const DrainBudget &) = delete; + DrainBudget &operator=(const DrainBudget &) = delete; + + private: + long long _saved; + }; + + // Counts a hook call in flight for its lifetime, as if one were stuck inside + // an engine. + class HookCallHold { + public: + explicit HookCallHold(Profiler *p) : _p(p) { + p->_hook_calls_inflight.fetch_add(1, std::memory_order_seq_cst); + } + ~HookCallHold() { + _p->_hook_calls_inflight.fetch_sub(1, std::memory_order_release); + } + HookCallHold(const HookCallHold &) = delete; + HookCallHold &operator=(const HookCallHold &) = delete; + + private: + Profiler *_p; + }; +}; + // qsort() requires a negative result when the first argument orders first. // A comparator that only returns 0/1 is sorted correctly by glibc's merge // sort, which tests `cmp <= 0`, but not by musl's smoothsort. @@ -102,6 +148,9 @@ class PerfEventsTestAccessor { static bool resizeEvents(PerfEvents &engine, int max_events) { return engine.resizeEvents(max_events); } + static Error sizeEventTable(PerfEvents &engine, int max_events) { + return engine.sizeEventTable(max_events); + } static PerfEvent *events() { return PerfEvents::_events; } static int maxEvents() { return PerfEvents::_max_events; } static PerfEvent *eventFor(int tid) { return PerfEvents::eventFor(tid); } @@ -361,6 +410,84 @@ TEST_F(PerfEventsResizeTest, FailedResizeAdvancesGenerationToEven) { EXPECT_EQ(0u, PerfEventsTestAccessor::gen() & 1); } +// A restart with an unchanged pid_max keeps the table and its fds without +// freezing thread unregistration, so a thread that exits during start() still +// closes its fd. A hook call is held in flight: a freeze would wait for it and +// time out, so success shows that none ran. +TEST_F(PerfEventsResizeTest, SameSizeRestartDoesNotFreezeUnregistration) { + Profiler *profiler = Profiler::instance(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + PerfEvent *table = PerfEventsTestAccessor::events(); + unsigned int gen = PerfEventsTestAccessor::gen(); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + Error error = Error::OK; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + error = PerfEventsTestAccessor::sizeEventTable(engine, 64); + } + + EXPECT_FALSE((bool)error); + EXPECT_EQ(table, PerfEventsTestAccessor::events()); + EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + +// A size change replaces the table only under a freeze. While a hook call is +// still in flight after the wait budget, start() fails retryably and keeps the +// old table and its slots, and unregistration is open again afterwards. +TEST_F(PerfEventsResizeTest, ResizeKeepsTableWhenHookCallsDoNotDrain) { + Profiler *profiler = Profiler::instance(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + int fd = -1, write_fd = -1; + void *page = nullptr; + armSlot(5, &fd, &write_fd, &page); + ASSERT_FALSE(HasFatalFailure()); + PerfEvent *table = PerfEventsTestAccessor::events(); + unsigned int gen = PerfEventsTestAccessor::gen(); + long long live = NativeMem::live(NM_PERF); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + Error error = Error::OK; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + error = PerfEventsTestAccessor::sizeEventTable(engine, 128); + } + + EXPECT_TRUE((bool)error); + EXPECT_STREQ("thread hook calls did not drain; retry start()", + error.message()); + EXPECT_EQ(table, PerfEventsTestAccessor::events()); + EXPECT_EQ(64, PerfEventsTestAccessor::maxEvents()); + EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); + EXPECT_EQ(fd, PerfEventsTestAccessor::slotFd(5)); + EXPECT_TRUE(ringMapped(page)); + EXPECT_EQ(live, NativeMem::live(NM_PERF)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + // The fixture only frees the table: release slot 5's fd and ring here. + engine.unregisterThread(5); + close(write_fd); +} + +// With no hook call in flight, a size change replaces the table and reopens +// unregistration. +TEST_F(PerfEventsResizeTest, ResizeReplacesTableWhenHookCallsDrain) { + Profiler *profiler = Profiler::instance(); + PerfEvents engine; + ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); + unsigned int gen = PerfEventsTestAccessor::gen(); + + Error error = PerfEventsTestAccessor::sizeEventTable(engine, 128); + + EXPECT_FALSE((bool)error); + EXPECT_EQ(128, PerfEventsTestAccessor::maxEvents()); + EXPECT_EQ(gen + 2, PerfEventsTestAccessor::gen()); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); +} + // A negative tid must not index before the table. The page before the table // is inaccessible, so an access through _events[-1] faults. TEST(PerfEventsTableTest, NegativeTidIsRejected) { diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 3b5ed34697..418dc07262 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -66,17 +66,16 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ``` Profiler::start() profiler.cpp:1555 - ThreadTablesFreeze both gates closed until the CPU engine - sizes its table (openThreadRegistration()); - unregistration reopened at the latest on - return. Waits (bounded, 1 s) for hook - calls; on timeout start() fails with a - retryable error before selecting engines - selectCpuEngine / selectWallEngine + _engines_ready cleared registration closed until the CPU engine + starts; unregistration stays open + selectCpuEngine / selectWallEngine engine pointers stored with release _jfr.start() SamplerPerf::primeClock() before any handler can fire Profiler::startCpuEngine() (EM_CPU) failure -> warning, continue _cpu_engine->start() + CTimer/PerfEvents: ThreadTablesFreeze only if the table size changes; + on a wait timeout (1 s) the engine fails retryably + and keeps its table CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads on failure: closeThreadRegistration() on success: openThreadRegistration() no-op for CTimer/PerfEvents @@ -119,8 +118,8 @@ Profiler::start() profiler.cpp:1555 and `CTimer::unregisterThread()` each release a slot with a per-slot CAS and neither frees the table; `PerfEvents::stop()` is a no-op and keeps its fds, which only `unregisterThread()` closes (under the fd CAS and the slot lock); - the wall engines and `ITimer` have no `unregisterThread()`. Only `start()` - replaces tables, under `ThreadTablesFreeze`. + the wall engines and `ITimer` have no `unregisterThread()`. Only a CPU + engine's `start()` replaces its table, under `ThreadTablesFreeze`. 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. @@ -149,7 +148,7 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: their table is sized and before they list threads, and which `Profiler::startCpuEngine()` sets once any CPU engine (including the table-less `ITimer`, `ITimerJvmti` and J9 engines) has started. It is - cleared by `stop()`, by a failed CPU engine start and by + cleared by `start()`, by `stop()`, by a failed CPU engine start and by `ThreadTablesFreeze`, so the gate is open iff the CPU engine started. Wall-only, noop and failed-CPU recordings leave it closed; no wall engine implements `registerThread()`, and one that ever does must open the gate @@ -160,16 +159,26 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: re-registers every live thread, so such a thread misses only the samples between the two passes (PROF-16359). - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only - while `start()` may replace the table (`ThreadTablesFreeze`). It stays - set while idle, because `PerfEvents::stop()` keeps per-thread fds open - and this call closes the fd of a thread that exits after `stop()`. + while `CTimer::sizeTimerTable()` or `PerfEvents::sizeEventTable()` + replaces the table (`ThreadTablesFreeze`), that is, when the table size + changes: the first start, or `pid_max` changed. A restart that keeps the + table never clears it. An exit that arrives while the gate is closed is + dropped, and if the thread is still listed when the engine lists + threads, its slot stays occupied after it dies (PROF-16359). The gate + stays set while idle, because `PerfEvents::stop()` keeps per-thread fds + open and this call closes the fd of a thread that exits after `stop()`. + - `start()` and `check()` replace `_cpu_engine`/`_wall_engine` while + unregistration is open, so they store the pointers with release and the + hook paths load them with acquire. An unregistration that reaches the + previous engine only releases that thread's own slot. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:988`). Every `Profiler::registerThread()`/`unregisterThread()` call counts itself in flight (`HookCallScope`) for as long as it may be inside an engine. Closing a gate -(`closeThreadRegistration()` in `stop()`, `ThreadTablesFreeze` in `start()`) waits +(`closeThreadRegistration()` in `stop()`, `ThreadTablesFreeze` around a table +replacement) waits for that count to reach zero, at most 1 s, so no `registerThread()` call is inside an engine once `stop()` tears it down, and no call at all while `start()` replaces its table. If the wait times out, the caller returns a retryable error and From 688e3683903d4fce30b61e9828d130244607a93d Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 09:18:52 -0400 Subject: [PATCH 20/26] PROF-16213: Defer PerfEvents table changes to PROF-16359 Restore perfEvents.h and perfEvents_linux.cpp to main and drop perfEvents_ut.cpp, so this change covers CTimer only. PerfEvents keeps one line: PerfEvents::start() opens thread registration before it lists threads. Without it, the registration gate in Profiler stays closed until start() returns, and a thread created while PerfEvents lists and sweeps threads is never registered for the recording. Comments and docs now describe PerfEvents as on main and point to PROF-16359. AGENTS.md lists PerfEvents::start() as a known exception to replacing a per-thread table only while its gate is closed. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 +- ddprof-lib/src/main/cpp/perfEvents.h | 29 -- ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 177 +------ ddprof-lib/src/main/cpp/profiler.cpp | 16 +- ddprof-lib/src/main/cpp/profiler.h | 15 +- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 510 ------------------- doc/architecture/ProfilingEngines.md | 40 +- 7 files changed, 65 insertions(+), 725 deletions(-) delete mode 100644 ddprof-lib/src/test/cpp/perfEvents_ut.cpp diff --git a/AGENTS.md b/AGENTS.md index 38e66f61ec..51ca99c429 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -285,7 +285,8 @@ for a temporary window. replace a table only while its gate is closed: closing a gate (`closeThreadRegistration()`, `ThreadTablesFreeze`) waits until no call is inside an engine. The wait is bounded; when it reports failure, tear nothing down, replace no - table and return a retryable error. `Profiler::startCpuEngine()` opens the + table and return a retryable error. Known exception: `PerfEvents::start()` still + replaces `_events` without a gate (PROF-16359). `Profiler::startCpuEngine()` opens the registration gate once any CPU engine started, so the gate is open iff it did; - an engine's `stop()` must tolerate a concurrent `unregisterThread()`: `stop()` closes only the registration gate, and unregistration stays open while the engines stop; diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h index d37329d3e8..560d214352 100644 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ b/ddprof-lib/src/main/cpp/perfEvents.h @@ -30,8 +30,6 @@ class PerfEventType; class StackContext; class PerfEvents : public Engine { - friend class PerfEventsTestAccessor; - private: static bool _enabled; static int _max_events; @@ -46,33 +44,6 @@ class PerfEvents : public Engine { static u64 readCounter(siginfo_t *siginfo, void *ucontext); // cppcheck-suppress unusedPrivateFunction static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - // Slot of tid in _events, or NULL when the table is NULL, being replaced, - // or tid is out of range. For readers in signal context (walkKernel(), - // resetBuffer()). - static PerfEvent *eventFor(int tid); - - // Generation of the (_events, _max_events) pair; odd while resizeEvents() - // replaces the table. eventFor() accepts a pair only from one even - // generation. - static unsigned int _events_gen; - static void beginTableUpdate(); - static void endTableUpdate(); - // Replaces the tid-indexed table with one of max_events slots, after - // releasing every fd and ring held by the old table. No-op when the size is - // unchanged. On allocation failure leaves _events == NULL and - // _max_events == -1 and returns false. Called only from sizeEventTable(), - // with the Profiler's registration gates closed. - bool resizeEvents(int max_events); - // Sizes the table for start(), replacing it under a - // Profiler::ThreadTablesFreeze only when the size changes. - Error sizeEventTable(int max_events); - -#ifdef UNIT_TEST - // Stores fd and ring page into slot tid of the current table (test only). - static void setSlotForTest(int tid, int fd, void *page); - // Reads back the fd of slot tid of the current table (test only). - static int slotFdForTest(int tid); -#endif public: Error check(Arguments &args); diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 23e5a630c5..4de0f227e0 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -574,7 +574,6 @@ class PerfEvent : public SpinLock { bool PerfEvents::_enabled = false; int PerfEvents::_max_events = -1; PerfEvent *PerfEvents::_events = NULL; -unsigned int PerfEvents::_events_gen = 0; PerfEventType *PerfEvents::_event_type = NULL; long PerfEvents::_interval; Ring PerfEvents::_ring; @@ -595,7 +594,7 @@ int PerfEvents::registerThread(int tid) { // It hasn't been started return 0; } - if (tid < 0 || tid >= _max_events) { + if (tid >= _max_events) { Log::warn("tid[%d] > pid_max[%d]. Restart profiler after changing pid_max", tid, _max_events); return -1; @@ -713,7 +712,7 @@ int PerfEvents::registerThread(int tid) { } void PerfEvents::unregisterThread(int tid) { - if (tid < 0 || tid >= _max_events) { + if (tid >= _max_events) { return; } @@ -912,96 +911,6 @@ Error PerfEvents::check(Arguments &args) { return Error::OK; } -void PerfEvents::beginTableUpdate() { - __atomic_store_n(&_events_gen, - __atomic_load_n(&_events_gen, __ATOMIC_RELAXED) + 1, - __ATOMIC_RELAXED); - // The odd generation is visible before any of the table stores. - __atomic_thread_fence(__ATOMIC_RELEASE); -} - -void PerfEvents::endTableUpdate() { - __atomic_store_n(&_events_gen, - __atomic_load_n(&_events_gen, __ATOMIC_RELAXED) + 1, - __ATOMIC_RELEASE); -} - -// The only writer of _events and _max_events outside tests. sizeEventTable() -// runs it under the Profiler's ThreadTablesFreeze, so the hook stays off the -// table; only the signal-path readers (eventFor()) run -// concurrently, and the generation bracket lets them reject a pair read while -// the table is replaced. -bool PerfEvents::resizeEvents(int max_events) { - if (max_events == _max_events) { - // Keeps the fds across stop()/start() when pid_max is unchanged. - return true; - } - // The table is the only record of which fd and ring belong to which tid. - // Release them while it still exists, otherwise neither this start() nor a - // later one can close them; start()'s sweep then re-registers the live - // threads into the new table. unregisterThread() excludes the signal-path - // readers with the slot lock, and is a no-op for an empty slot (fd 0 or -1, - // no page). The walk touches every slot, but runs only when pid_max changes - // or after a failed allocation. - if (_events != NULL) { - for (int tid = 0; tid < _max_events; tid++) { - PerfEvents::unregisterThread(tid); - } - } - PerfEvent *old_events = _events; - int old_max = _max_events; - beginTableUpdate(); - __atomic_store_n(&_events, (PerfEvent *)NULL, __ATOMIC_RELAXED); - __atomic_store_n(&_max_events, -1, __ATOMIC_RELAXED); - free(old_events); - // Record the decrement after the free, consistent with the other decrement - // sites; the guard keeps an earlier failed allocation from being released. - if (old_events != NULL && old_max > 0) { - NativeMem::record(NM_PERF, - -(long long)((size_t)old_max * sizeof(PerfEvent))); - } - // On failure the pair stays NULL / -1: registerThread() treats the engine as - // not started, and the tid bound checks in unregisterThread() and eventFor() - // reject every tid, so nothing indexes the NULL table. The next start() - // retries the allocation. - PerfEvent *events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); - if (events != NULL) { - __atomic_store_n(&_events, events, __ATOMIC_RELAXED); - __atomic_store_n(&_max_events, max_events, __ATOMIC_RELAXED); - NativeMem::record(NM_PERF, - (long long)((size_t)max_events * sizeof(PerfEvent))); - } - endTableUpdate(); - return events != NULL; -} - -// A restart with an unchanged pid_max keeps the table and never closes thread -// unregistration, so an exit during start() still closes its fd. Only an -// actual replacement freezes the hook calls; if one is still inside the table -// after the wait budget, start() fails retryably and keeps the old table. -Error PerfEvents::sizeEventTable(int max_events) { - if (max_events == _max_events) { - return Error::OK; - } - Profiler::ThreadTablesFreeze freeze(Profiler::instance()); - if (!freeze.drained()) { - return Error("thread hook calls did not drain; retry start()"); - } - if (!resizeEvents(max_events)) { - return Error("Failed to allocate perf event table"); - } - return Error::OK; -} - -#ifdef UNIT_TEST -void PerfEvents::setSlotForTest(int tid, int fd, void *page) { - _events[tid]._fd = fd; - _events[tid]._page = (struct perf_event_mmap_page *)page; -} - -int PerfEvents::slotFdForTest(int tid) { return _events[tid]._fd; } -#endif - Error PerfEvents::start(Arguments &args) { _event_type = PerfEventType::forName(args._event == NULL ? EVENT_CPU : args._event); @@ -1045,13 +954,24 @@ Error PerfEvents::start(Arguments &args) { (_ring != RING_USER || _cstack == CSTACK_DEFAULT || _cstack == CSTACK_LBR); - // Account the per-thread PerfEvent array under NM_PERF, alongside the ring - // mappings. resizeEvents() closes the fds of the old table before freeing - // it; on allocation failure the table stays empty and the next start() - // retries. - Error error = sizeEventTable(OS::getMaxThreadId()); - if (error) { - return error; + int max_events = OS::getMaxThreadId(); + if (max_events != _max_events) { + // Account the per-thread PerfEvent array under NM_PERF, alongside the ring + // mappings. Guard on non-null so a prior calloc failure isn't mis-accounted. + // Record the decrement after the free (consistent with the other decrement + // sites); capture the old size first since _max_events is overwritten below. + long long old_bytes = + (_events != NULL) ? (long long)((size_t)_max_events * sizeof(PerfEvent)) : 0; + free(_events); + if (old_bytes > 0) { + NativeMem::record(NM_PERF, -old_bytes); + } + _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); + _max_events = max_events; + if (_events != NULL) { + NativeMem::record(NM_PERF, + (long long)((size_t)max_events * sizeof(PerfEvent))); + } } OS::installSignalHandler(SIGPROF, signalHandler); @@ -1108,25 +1028,6 @@ Error PerfEvents::start(Arguments &args) { free(threads); - if (err == 0) { - // The sweep above closes the fd of every tid missing from the snapshot, - // including a thread the hook registered after the snapshot was taken. - // Such a thread was already live, so a second listing sees it; - // registerThread() skips tids that kept their fd and its 0 -> -1 CAS - // resolves races with the hook. Threads started after the sweep are left - // to the hook. Best effort: never fails start() (PROF-16359). - ThreadList *rearm = OS::listThreads(); - while (rearm->hasNext()) { - int tid = rearm->next(); - int rc = registerThread(tid); - if (rc != 0 && rc != ESRCH && rc != -1) { - Log::debug("perfEvents: re-registering TID %d failed: %s", tid, - strerror(rc)); - } - } - delete rearm; - } - if (err != 0) { *_pthread_entry = (void *)pthread_setspecific; Profiler::instance()->switchThreadEvents(JVMTI_DISABLE); @@ -1162,9 +1063,9 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, return 0; } - PerfEvent *event = eventFor(tid); - if (event == NULL || !event->tryLock()) { - return 0; // no table, or the event is being destroyed + PerfEvent *event = &_events[tid]; + if (!event->tryLock()) { + return 0; // the event is being destroyed } int depth = 0; @@ -1250,36 +1151,10 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, return depth; } -// walkKernel() and resetBuffer() run from the SIGPROF handler, including for -// signals from fds left armed by an earlier recording. start() can leave the -// table NULL (_max_events == -1) after a failed allocation, so they reach the -// table only through this check. The generation check makes the bound always -// belong to the table it validates, including while resizeEvents() replaces -// the table; the reader never spins, it returns NULL instead, so a sample on -// the start() thread inside that window loses its kernel frames. A slot -// pointer returned before resizeEvents() frees the table is still unsafe to -// use afterwards (PROF-16359). -PerfEvent *PerfEvents::eventFor(int tid) { - unsigned int gen = __atomic_load_n(&_events_gen, __ATOMIC_ACQUIRE); - if (gen & 1) { - return NULL; // the table is being replaced - } - PerfEvent *events = __atomic_load_n(&_events, __ATOMIC_RELAXED); - int max_events = __atomic_load_n(&_max_events, __ATOMIC_RELAXED); - __atomic_thread_fence(__ATOMIC_ACQUIRE); - if (__atomic_load_n(&_events_gen, __ATOMIC_RELAXED) != gen) { - return NULL; // the table changed while it was read - } - if (events == NULL || tid < 0 || tid >= max_events) { - return NULL; - } - return &events[tid]; -} - void PerfEvents::resetBuffer(int tid) { - PerfEvent *event = eventFor(tid); - if (event == NULL || !event->tryLock()) { - return; // no table, or the event is being destroyed + PerfEvent *event = &_events[tid]; + if (!event->tryLock()) { + return; // the event is being destroyed } struct perf_event_mmap_page *page = event->_page; diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index de2d339ca4..96ed3a7931 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -320,8 +320,9 @@ void Profiler::unregisterThreadInEngines(int tid) { #endif // start() and check() may replace the engines concurrently. Either engine // is safe to call: unregisterThread() only releases the exiting thread's own - // slot, bound-checked against the engine's current table, and a table being - // replaced is behind the gate above. + // slot, bound-checked against the engine's current table, and a CTimer table + // being replaced is behind the gate above. PerfEvents replaces its table + // without the gate (PROF-16359). __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE) ->unregisterThread(tid); __atomic_load_n(&_instance->_wall_engine, __ATOMIC_ACQUIRE) @@ -2009,11 +2010,12 @@ Error Profiler::stop() { // JVMTI ThreadEnd and native thread exits may still reach unregisterThread() // while the engines stop. That is safe per engine: CTimer::stop() and // CTimer::unregisterThread() each release a slot with a per-slot CAS and - // neither frees the table; PerfEvents::stop() is a no-op and the slot is - // released under its fd CAS and slot lock; wall engines and ITimer have no - // unregisterThread(). Only a CPU engine's start() replaces its table, under - // ThreadTablesFreeze. Unregistration stays open so PerfEvents can close the - // fd of a thread that exits after stop(). + // neither frees the table; PerfEvents::stop() is a no-op, so it touches no + // slot an unregistration releases; wall engines and ITimer have no + // unregisterThread(). Only a CPU engine's start() replaces its table: + // CTimer under ThreadTablesFreeze, PerfEvents without it (PROF-16359). + // Unregistration stays open so PerfEvents can close the fd of a thread that + // exits after stop(). if (!closeThreadRegistration()) { // Same retry contract as the SignalInflight timeout above: state stays // RUNNING, no engine stop has run, and a retry re-runs every step. diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index afa8ee3293..01d0dbc667 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -293,10 +293,11 @@ class alignas(alignof(SpinLock)) Profiler { public: // Keeps Profiler::registerThread() and unregisterThread() off the engines - // while a CPU engine replaces its per-thread table. CTimer and PerfEvents - // construct it in start() only when the table size changes (pid_max changed, - // or no table yet): a thread exit skipped while it is alive is dropped, so it - // must not span a restart that keeps the table. Both gates are set again by + // while a CPU engine replaces its per-thread table. CTimer constructs it in + // start() only when the table size changes (pid_max changed, or no table + // yet): a thread exit skipped while it is alive is dropped, so it must not + // span a restart that keeps the table. PerfEvents::start() still replaces its + // table without it (PROF-16359). Both gates are set again by // openThreadRegistration() once the table is sized; the destructor reopens // unregistration at the latest. An exit skipped in this window is cleaned up // only if the thread is gone when the engine lists threads; a thread that is @@ -466,9 +467,9 @@ class alignas(alignof(SpinLock)) Profiler { * engine that needs registerThread() must open the gate itself. For CTimer * this misses no thread: the per-slot CAS resolves the hook and start() * registering one tid. PerfEvents::start()'s stale-fd sweep may close the - * fd of a tid the hook registered after the thread list was taken; a - * second listing pass re-registers it, so such a thread misses only the - * samples between the two passes (PROF-16359). Engines without + * fd of a tid the hook registered after the thread list was taken, and + * such a thread is then not sampled for the recording (PROF-16359). + * Engines without * registerThread() never need the gate. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp deleted file mode 100644 index e362c361df..0000000000 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ /dev/null @@ -1,510 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#ifdef __linux__ - -#include "arguments.h" -#include "common.h" -#include "nativeMem.h" -#include "perfEvents.h" -#include "profiler.h" - -#include -#include -#include -#include -#include -#include -#include - -// The failure-path tests force calloc to fail with a negative count, which -// makes count * size overflow. The CI sanitizer jobs run gtest binaries -// without *SAN_OPTIONS, so allocator_may_return_null defaults to 0 and the -// sanitizer reports that overflow as a fatal error instead of returning NULL. -#if defined(ASAN_ENABLED) || defined(TSAN_ENABLED) -#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() \ - GTEST_SKIP() << "sanitizer allocators abort on calloc overflow" -#else -#define SKIP_IF_CALLOC_OVERFLOW_ABORTS() -#endif - -static int compare(int a, int b) { return PerfEvents::compareTids(&a, &b); } - -// The parts of the Profiler's thread-hook gates the table-sizing tests need. -// This binary defines its own copy of the friend accessor; ctimer_ut.cpp has -// the full one. -class ProfilerTestAccessor { -public: - static bool enginesReady(Profiler *p) { - return p->_engines_ready.load(std::memory_order_acquire); - } - static bool threadTablesStable(Profiler *p) { - return p->_thread_tables_stable.load(std::memory_order_acquire); - } - - // Sets the hook-call wait budget for its lifetime and restores the previous - // one on destruction, including when a test assertion returns early. - class DrainBudget { - public: - explicit DrainBudget(long long ns) : _saved(Profiler::_hook_drain_timeout_ns) { - Profiler::_hook_drain_timeout_ns = ns; - } - ~DrainBudget() { Profiler::_hook_drain_timeout_ns = _saved; } - DrainBudget(const DrainBudget &) = delete; - DrainBudget &operator=(const DrainBudget &) = delete; - - private: - long long _saved; - }; - - // Counts a hook call in flight for its lifetime, as if one were stuck inside - // an engine. - class HookCallHold { - public: - explicit HookCallHold(Profiler *p) : _p(p) { - p->_hook_calls_inflight.fetch_add(1, std::memory_order_seq_cst); - } - ~HookCallHold() { - _p->_hook_calls_inflight.fetch_sub(1, std::memory_order_release); - } - HookCallHold(const HookCallHold &) = delete; - HookCallHold &operator=(const HookCallHold &) = delete; - - private: - Profiler *_p; - }; -}; - -// qsort() requires a negative result when the first argument orders first. -// A comparator that only returns 0/1 is sorted correctly by glibc's merge -// sort, which tests `cmp <= 0`, but not by musl's smoothsort. -TEST(PerfEventsCompareTidsTest, FollowsQsortContract) { - EXPECT_LT(compare(1, 2), 0); - EXPECT_GT(compare(2, 1), 0); - EXPECT_EQ(compare(7, 7), 0); - // Extremes must not overflow, as a subtraction-based comparator would. - EXPECT_LT(compare(INT_MIN, INT_MAX), 0); - EXPECT_GT(compare(INT_MAX, INT_MIN), 0); - EXPECT_LT(compare(-1, 0), 0); -} - -// /proc/self/task lists threads in creation order, so once tids are reused a -// later thread can carry a lower tid. PerfEvents::start() matches tids against -// the sorted list in ascending order and skips every thread after the first -// out-of-order entry, so the sort must hold for such input on every libc. -TEST(PerfEventsCompareTidsTest, SortsCreationOrderedTidsWithReuse) { - int tids[] = {29, 5001, 5002, 5003, 5004, 5005, 201, 202, 203, 9001, 9002, 9003}; - const int n = sizeof(tids) / sizeof(tids[0]); - - qsort(tids, n, sizeof(int), PerfEvents::compareTids); - - for (int i = 1; i < n; i++) { - EXPECT_LT(tids[i - 1], tids[i]) << "unsorted at index " << i; - } -} - -// Swaps PerfEvents' table, bound and ring mode for a test and restores them on -// destruction, including when an assertion returns early. -class PerfEventsTestAccessor { -public: - class TableSwap { - public: - TableSwap(PerfEvent *events, int max_events, Ring ring) - : _events(PerfEvents::_events), _max_events(PerfEvents::_max_events), - _ring(PerfEvents::_ring) { - PerfEvents::_events = events; - PerfEvents::_max_events = max_events; - PerfEvents::_ring = ring; - } - ~TableSwap() { - PerfEvents::_events = _events; - PerfEvents::_max_events = _max_events; - PerfEvents::_ring = _ring; - } - TableSwap(const TableSwap &) = delete; - TableSwap &operator=(const TableSwap &) = delete; - - private: - PerfEvent *_events; - int _max_events; - Ring _ring; - }; - - static bool resizeEvents(PerfEvents &engine, int max_events) { - return engine.resizeEvents(max_events); - } - static Error sizeEventTable(PerfEvents &engine, int max_events) { - return engine.sizeEventTable(max_events); - } - static PerfEvent *events() { return PerfEvents::_events; } - static int maxEvents() { return PerfEvents::_max_events; } - static PerfEvent *eventFor(int tid) { return PerfEvents::eventFor(tid); } - static unsigned int gen() { return PerfEvents::_events_gen; } - static void setGen(unsigned int gen) { PerfEvents::_events_gen = gen; } - static void setSlot(int tid, int fd, void *page) { - PerfEvents::setSlotForTest(tid, fd, page); - } - static int slotFd(int tid) { return PerfEvents::slotFdForTest(tid); } - - // Hands the current table to the caller without freeing it, so a test can - // start from an empty table and later reinstate the original one. - static void detach() { - PerfEvents::_events = nullptr; - PerfEvents::_max_events = -1; - } - - // Frees the test-owned table and reinstates the detached one, without - // touching the gauge (the fixture restores that separately). - static void reattach(PerfEvent *events, int max_events) { - free(PerfEvents::_events); - PerfEvents::_events = events; - PerfEvents::_max_events = max_events; - } -}; - -// After a failed table allocation, start() leaves _events NULL with -// _max_events == -1 while fds from an earlier recording may still fire -// SIGPROF. The signal-path readers must not index the NULL table, and -// unregisterThread() must reject every tid. -TEST(PerfEventsTableTest, SignalPathReadersSkipNullTable) { - PerfEventsTestAccessor::TableSwap swap(nullptr, -1, RING_KERNEL); - const void *callchain[4]; - - for (int tid : {0, 1, 4096}) { - EXPECT_EQ(0, PerfEvents::walkKernel(tid, callchain, 4, nullptr)); - PerfEvents::resetBuffer(tid); - PerfEvents engine; - engine.unregisterThread(tid); - } -} - -// The signal-path readers check the tid against the bound before indexing. -// The table points at an inaccessible page, so any access through it faults. -TEST(PerfEventsTableTest, SignalPathReadersRejectOutOfRangeTid) { - void *page = mmap(nullptr, 1 << 16, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); - ASSERT_NE(MAP_FAILED, page); - { - PerfEventsTestAccessor::TableSwap swap( - reinterpret_cast(page), 0, RING_KERNEL); - const void *callchain[4]; - - for (int tid : {-1, 0, 1}) { - EXPECT_EQ(0, PerfEvents::walkKernel(tid, callchain, 4, nullptr)); - PerfEvents::resetBuffer(tid); - } - } - munmap(page, 1 << 16); -} - -// Runs resizeEvents() against an empty table and an NM_PERF gauge starting at -// zero, and puts the original table, generation and gauge back afterwards. -class PerfEventsResizeTest : public ::testing::Test { -protected: - PerfEvent *_saved_events; - int _saved_max; - unsigned int _saved_gen; - long long _saved_live; - size_t _page_size; - - void SetUp() override { - _saved_events = PerfEventsTestAccessor::events(); - _saved_max = PerfEventsTestAccessor::maxEvents(); - _saved_gen = PerfEventsTestAccessor::gen(); - _saved_live = NativeMem::live(NM_PERF); - _page_size = (size_t)sysconf(_SC_PAGESIZE); - PerfEventsTestAccessor::detach(); - NativeMem::setLive(NM_PERF, 0); - } - - void TearDown() override { - PerfEventsTestAccessor::reattach(_saved_events, _saved_max); - PerfEventsTestAccessor::setGen(_saved_gen); - NativeMem::setLive(NM_PERF, _saved_live); - } - - // Puts a pipe read end and a 2-page anonymous mapping into slot tid, and - // accounts the ring the way registerThread() does. Returns the write end - // through *write_fd. - void armSlot(int tid, int *read_fd, int *write_fd, void **page) { - int fds[2]; - ASSERT_EQ(0, pipe(fds)); - *read_fd = fds[0]; - *write_fd = fds[1]; - *page = mmap(nullptr, 2 * _page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(MAP_FAILED, *page); - *(uint64_t *)*page = RING_SENTINEL; - PerfEventsTestAccessor::setSlot(tid, *read_fd, *page); - NativeMem::record(NM_PERF, (long long)(2 * _page_size)); - } - - // Reports whether the ring armSlot() mapped at page is still mapped. Once - // the ring is unmapped, the allocator may map new memory at the same - // address: musl's malloc does when resizeEvents() allocates the new table. - // So the check reads back the sentinel armSlot() wrote instead of only - // probing the address. The read goes through a pipe, which fails with - // EFAULT instead of faulting when the page is unmapped or inaccessible. - static bool ringMapped(void *page) { - int fds[2]; - if (pipe(fds) != 0) { - ADD_FAILURE() << "pipe() failed: " << errno; - return true; - } - uint64_t value = 0; - bool readable = write(fds[1], page, sizeof(value)) == sizeof(value) && - read(fds[0], &value, sizeof(value)) == sizeof(value); - close(fds[0]); - close(fds[1]); - return readable && value == RING_SENTINEL; - } - - static constexpr uint64_t RING_SENTINEL = 0x5a5a5a5a5a5a5a5aULL; -}; - -// A failed allocation leaves a NULL table with a -1 bound, not a NULL table -// with a stale bound, releases the old table's bytes from the gauge, and a -// later resize recovers. -TEST_F(PerfEventsResizeTest, FailedAllocationLeavesNoTable) { - SKIP_IF_CALLOC_OVERFLOW_ABORTS(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 1024)); - ASSERT_NE(nullptr, PerfEventsTestAccessor::events()); - EXPECT_EQ(1024, PerfEventsTestAccessor::maxEvents()); - long long one = NativeMem::live(NM_PERF); - EXPECT_GT(one, 0); - - EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); - EXPECT_EQ(nullptr, PerfEventsTestAccessor::events()); - EXPECT_EQ(-1, PerfEventsTestAccessor::maxEvents()); - EXPECT_EQ(0, NativeMem::live(NM_PERF)); - EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(0)); - // The failure state reads as "not started" and rejects every tid. - EXPECT_EQ(0, engine.registerThread(5)); - engine.unregisterThread(5); - - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 1024)); - EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(1023)); - EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(1024)); - EXPECT_EQ(one, NativeMem::live(NM_PERF)); -} - -// Replacing the table closes every fd and unmaps every ring the old table -// holds: once it is freed nothing could reach them. The fd check runs right -// after the resize, before anything else in the test opens a file, so the fd -// number cannot have been reused (gtest is single-threaded here). The ring -// address can be: see ringMapped(). -TEST_F(PerfEventsResizeTest, ResizeReleasesOldSlots) { - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - long long base = NativeMem::live(NM_PERF); - int fd = -1, write_fd = -1; - void *page = nullptr; - armSlot(5, &fd, &write_fd, &page); - ASSERT_FALSE(HasFatalFailure()); - ASSERT_EQ(base + (long long)(2 * _page_size), NativeMem::live(NM_PERF)); - - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 128)); - errno = 0; - int fcntl_rc = fcntl(fd, F_GETFD); - int fcntl_errno = errno; - bool mapped = ringMapped(page); - - EXPECT_EQ(-1, fcntl_rc); - EXPECT_EQ(EBADF, fcntl_errno); - EXPECT_FALSE(mapped); - EXPECT_EQ(2 * base, NativeMem::live(NM_PERF)); - EXPECT_EQ(0, PerfEventsTestAccessor::slotFd(5)); - close(write_fd); -} - -// The failure path releases the old table's slots too. -TEST_F(PerfEventsResizeTest, FailedResizeReleasesOldSlots) { - SKIP_IF_CALLOC_OVERFLOW_ABORTS(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - int fd = -1, write_fd = -1; - void *page = nullptr; - armSlot(5, &fd, &write_fd, &page); - ASSERT_FALSE(HasFatalFailure()); - - EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); - errno = 0; - int fcntl_rc = fcntl(fd, F_GETFD); - int fcntl_errno = errno; - bool mapped = ringMapped(page); - - EXPECT_EQ(-1, fcntl_rc); - EXPECT_EQ(EBADF, fcntl_errno); - EXPECT_FALSE(mapped); - EXPECT_EQ(0, NativeMem::live(NM_PERF)); - close(write_fd); -} - -// An unchanged size keeps the table, its fds and the gauge, so fds survive a -// stop()/start() when pid_max did not change. -TEST_F(PerfEventsResizeTest, SameSizeKeepsTable) { - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - int fds[2]; - ASSERT_EQ(0, pipe(fds)); - PerfEventsTestAccessor::setSlot(3, fds[0], nullptr); - PerfEvent *table = PerfEventsTestAccessor::events(); - unsigned int gen = PerfEventsTestAccessor::gen(); - long long live = NativeMem::live(NM_PERF); - - bool resized = PerfEventsTestAccessor::resizeEvents(engine, 64); - int fcntl_rc = fcntl(fds[0], F_GETFD); - - EXPECT_TRUE(resized); - EXPECT_EQ(table, PerfEventsTestAccessor::events()); - EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); - EXPECT_EQ(live, NativeMem::live(NM_PERF)); - EXPECT_NE(-1, fcntl_rc); - PerfEventsTestAccessor::setSlot(3, 0, nullptr); - close(fds[0]); - close(fds[1]); -} - -// eventFor() rejects the table while the generation is odd (a replacement in -// progress), and every resize, failed or not, advances the generation by two -// and leaves it even. -TEST_F(PerfEventsResizeTest, EventForRejectsOddGenerationAndResizeAdvancesIt) { - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - unsigned int g = PerfEventsTestAccessor::gen(); - EXPECT_EQ(0u, g & 1); - EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(3)); - - PerfEventsTestAccessor::setGen(g + 1); - EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(3)); - PerfEventsTestAccessor::setGen(g); - EXPECT_NE(nullptr, PerfEventsTestAccessor::eventFor(3)); - - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 128)); - EXPECT_EQ(g + 2, PerfEventsTestAccessor::gen()); -} - -TEST_F(PerfEventsResizeTest, FailedResizeAdvancesGenerationToEven) { - SKIP_IF_CALLOC_OVERFLOW_ABORTS(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - unsigned int g = PerfEventsTestAccessor::gen(); - EXPECT_FALSE(PerfEventsTestAccessor::resizeEvents(engine, -1)); - EXPECT_EQ(g + 2, PerfEventsTestAccessor::gen()); - EXPECT_EQ(0u, PerfEventsTestAccessor::gen() & 1); -} - -// A restart with an unchanged pid_max keeps the table and its fds without -// freezing thread unregistration, so a thread that exits during start() still -// closes its fd. A hook call is held in flight: a freeze would wait for it and -// time out, so success shows that none ran. -TEST_F(PerfEventsResizeTest, SameSizeRestartDoesNotFreezeUnregistration) { - Profiler *profiler = Profiler::instance(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - PerfEvent *table = PerfEventsTestAccessor::events(); - unsigned int gen = PerfEventsTestAccessor::gen(); - ProfilerTestAccessor::DrainBudget budget(100000000LL); - - Error error = Error::OK; - { - ProfilerTestAccessor::HookCallHold hold(profiler); - error = PerfEventsTestAccessor::sizeEventTable(engine, 64); - } - - EXPECT_FALSE((bool)error); - EXPECT_EQ(table, PerfEventsTestAccessor::events()); - EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); - EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); -} - -// A size change replaces the table only under a freeze. While a hook call is -// still in flight after the wait budget, start() fails retryably and keeps the -// old table and its slots, and unregistration is open again afterwards. -TEST_F(PerfEventsResizeTest, ResizeKeepsTableWhenHookCallsDoNotDrain) { - Profiler *profiler = Profiler::instance(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - int fd = -1, write_fd = -1; - void *page = nullptr; - armSlot(5, &fd, &write_fd, &page); - ASSERT_FALSE(HasFatalFailure()); - PerfEvent *table = PerfEventsTestAccessor::events(); - unsigned int gen = PerfEventsTestAccessor::gen(); - long long live = NativeMem::live(NM_PERF); - ProfilerTestAccessor::DrainBudget budget(100000000LL); - - Error error = Error::OK; - { - ProfilerTestAccessor::HookCallHold hold(profiler); - error = PerfEventsTestAccessor::sizeEventTable(engine, 128); - } - - EXPECT_TRUE((bool)error); - EXPECT_STREQ("thread hook calls did not drain; retry start()", - error.message()); - EXPECT_EQ(table, PerfEventsTestAccessor::events()); - EXPECT_EQ(64, PerfEventsTestAccessor::maxEvents()); - EXPECT_EQ(gen, PerfEventsTestAccessor::gen()); - EXPECT_EQ(fd, PerfEventsTestAccessor::slotFd(5)); - EXPECT_TRUE(ringMapped(page)); - EXPECT_EQ(live, NativeMem::live(NM_PERF)); - EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); - EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); - // The fixture only frees the table: release slot 5's fd and ring here. - engine.unregisterThread(5); - close(write_fd); -} - -// With no hook call in flight, a size change replaces the table and reopens -// unregistration. -TEST_F(PerfEventsResizeTest, ResizeReplacesTableWhenHookCallsDrain) { - Profiler *profiler = Profiler::instance(); - PerfEvents engine; - ASSERT_TRUE(PerfEventsTestAccessor::resizeEvents(engine, 64)); - unsigned int gen = PerfEventsTestAccessor::gen(); - - Error error = PerfEventsTestAccessor::sizeEventTable(engine, 128); - - EXPECT_FALSE((bool)error); - EXPECT_EQ(128, PerfEventsTestAccessor::maxEvents()); - EXPECT_EQ(gen + 2, PerfEventsTestAccessor::gen()); - EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); -} - -// A negative tid must not index before the table. The page before the table -// is inaccessible, so an access through _events[-1] faults. -TEST(PerfEventsTableTest, NegativeTidIsRejected) { - size_t page_size = (size_t)sysconf(_SC_PAGESIZE); - char *base = (char *)mmap(nullptr, 3 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(MAP_FAILED, (void *)base); - ASSERT_EQ(0, mprotect(base, page_size, PROT_NONE)); - { - PerfEventsTestAccessor::TableSwap swap( - reinterpret_cast(base + page_size), 4, RING_KERNEL); - PerfEvents engine; - EXPECT_EQ(-1, engine.registerThread(-1)); - engine.unregisterThread(-1); - EXPECT_EQ(nullptr, PerfEventsTestAccessor::eventFor(-1)); - } - munmap(base, 3 * page_size); -} - -#endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 418dc07262..227d5ceb6c 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -73,9 +73,10 @@ Profiler::start() profiler.cpp:1555 SamplerPerf::primeClock() before any handler can fire Profiler::startCpuEngine() (EM_CPU) failure -> warning, continue _cpu_engine->start() - CTimer/PerfEvents: ThreadTablesFreeze only if the table size changes; - on a wait timeout (1 s) the engine fails retryably - and keeps its table + CTimer: ThreadTablesFreeze only if the table size changes; + on a wait timeout (1 s) it fails retryably and keeps + its table + PerfEvents: replaces its table without the freeze (PROF-16359) CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads on failure: closeThreadRegistration() on success: openThreadRegistration() no-op for CTimer/PerfEvents @@ -117,9 +118,9 @@ Profiler::start() profiler.cpp:1555 may still reach `unregisterThread()` while the engines stop. `CTimer::stop()` and `CTimer::unregisterThread()` each release a slot with a per-slot CAS and neither frees the table; `PerfEvents::stop()` is a no-op and keeps its fds, - which only `unregisterThread()` closes (under the fd CAS and the slot lock); - the wall engines and `ITimer` have no `unregisterThread()`. Only a CPU - engine's `start()` replaces its table, under `ThreadTablesFreeze`. + which only `unregisterThread()` closes; the wall engines and `ITimer` have + no `unregisterThread()`. Only a CPU engine's `start()` replaces its table: + `CTimer` under `ThreadTablesFreeze`, `PerfEvents` without it (PROF-16359). 4. Then, in order: `_alloc_engine->stop()`, reference-chain teardown, `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. @@ -155,14 +156,14 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: itself. While the gate is closed, `registerThread()` returns before it touches the in-flight counter. - `PerfEvents::start()`'s stale-fd sweep may close the fd of a tid the hook - registered after the thread list was taken; a second listing pass then - re-registers every live thread, so such a thread misses only the samples - between the two passes (PROF-16359). + registered after the thread list was taken, and such a thread is then not + sampled for the recording (PROF-16359). - `unregisterThread()` needs `_thread_tables_stable`, which is cleared only - while `CTimer::sizeTimerTable()` or `PerfEvents::sizeEventTable()` - replaces the table (`ThreadTablesFreeze`), that is, when the table size - changes: the first start, or `pid_max` changed. A restart that keeps the - table never clears it. An exit that arrives while the gate is closed is + while `CTimer::sizeTimerTable()` replaces the timer table + (`ThreadTablesFreeze`), that is, when the table size changes: the first + start, or `pid_max` changed. A restart that keeps the table never clears + it. `PerfEvents::start()` replaces its table without clearing it + (PROF-16359). An exit that arrives while the gate is closed is dropped, and if the thread is still listed when the engine lists threads, its slot stays occupied after it dies (PROF-16359). The gate stays set while idle, because `PerfEvents::stop()` keeps per-thread fds @@ -173,7 +174,7 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: previous engine only releases that thread's own slot. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` - (`perfEvents_linux.cpp:988`). + (`perfEvents_linux.cpp:980`). Every `Profiler::registerThread()`/`unregisterThread()` call counts itself in flight (`HookCallScope`) for as long as it may be inside an engine. Closing a gate @@ -302,17 +303,16 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1059`): fds stay open across +- **`stop()` is a no-op** (`perfEvents_linux.cpp:1051`): fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. The array is counted at its full allocated size, which is mostly untouched address space (see Profiler Self-Accounting). -- **Table replacement**: only `resizeEvents()`, called from `start()` when - `pid_max` changed or after a failed allocation, replaces `_events`. It closes - every fd and unmaps every ring of the old table before freeing it, and - brackets the replacement with a seqlock generation so the signal-path - readers (`eventFor()`) never pair a table with another table's bound. +- **Table replacement**: `start()` frees and reallocates `_events` when + `pid_max` changed. It does not close the old table's fds or rings, and it + does not exclude thread unregistration or the signal-path readers + (`walkKernel()`, `resetBuffer()`) while it does so (PROF-16359). - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this engine (`profiler.cpp:1725`, `1745`). From 08c964e8b438f3545b52d88244a39390db5d275c Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 10:06:37 -0400 Subject: [PATCH 21/26] PROF-16213: Restore perfEvents_ut.cpp from main The previous commit deleted perfEvents_ut.cpp instead of restoring it to main. The file exists on main with the PerfEventsCompareTidsTest tests, so the deletion would have removed them. This change no longer touches the file. Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 56 +++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 ddprof-lib/src/test/cpp/perfEvents_ut.cpp diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp new file mode 100644 index 0000000000..59bb7c6915 --- /dev/null +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -0,0 +1,56 @@ +/* + * Copyright 2026, Datadog, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#ifdef __linux__ + +#include "perfEvents.h" + +#include +#include + +static int compare(int a, int b) { return PerfEvents::compareTids(&a, &b); } + +// qsort() requires a negative result when the first argument orders first. +// A comparator that only returns 0/1 is sorted correctly by glibc's merge +// sort, which tests `cmp <= 0`, but not by musl's smoothsort. +TEST(PerfEventsCompareTidsTest, FollowsQsortContract) { + EXPECT_LT(compare(1, 2), 0); + EXPECT_GT(compare(2, 1), 0); + EXPECT_EQ(compare(7, 7), 0); + // Extremes must not overflow, as a subtraction-based comparator would. + EXPECT_LT(compare(INT_MIN, INT_MAX), 0); + EXPECT_GT(compare(INT_MAX, INT_MIN), 0); + EXPECT_LT(compare(-1, 0), 0); +} + +// /proc/self/task lists threads in creation order, so once tids are reused a +// later thread can carry a lower tid. PerfEvents::start() matches tids against +// the sorted list in ascending order and skips every thread after the first +// out-of-order entry, so the sort must hold for such input on every libc. +TEST(PerfEventsCompareTidsTest, SortsCreationOrderedTidsWithReuse) { + int tids[] = {29, 5001, 5002, 5003, 5004, 5005, 201, 202, 203, 9001, 9002, 9003}; + const int n = sizeof(tids) / sizeof(tids[0]); + + qsort(tids, n, sizeof(int), PerfEvents::compareTids); + + for (int i = 1; i < n; i++) { + EXPECT_LT(tids[i - 1], tids[i]) << "unsorted at index " << i; + } +} + +#endif // __linux__ From 9de032e88afcbbf910300a3c207f7189e7a7d7f0 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 13:28:17 -0400 Subject: [PATCH 22/26] PROF-16213: Admit only the CPU engine through the registration gate The registration gate is open iff the CPU engine started, but registerThreadInEngines() also called the wall engine through it, and the docs told a future wall engine that needs registerThread() to open the gate itself. Doing that would admit a CPU engine whose start() failed, and a started CPU engine already opened the gate for a wall engine that did not start. No wall engine overrides Engine::registerThread(), so the wall call did nothing. Registration now reaches only the CPU engine and returns its result; no caller reads the value. The docs and AGENTS.md now say a wall engine that needs registerThread() needs its own gate. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++- ddprof-lib/src/main/cpp/profiler.cpp | 14 +++++---- ddprof-lib/src/main/cpp/profiler.h | 13 +++++---- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 42 ++++++++++++++++++++++++--- doc/architecture/ProfilingEngines.md | 11 ++++--- 5 files changed, 64 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 51ca99c429..895e62996e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -287,7 +287,9 @@ for a temporary window. engine. The wait is bounded; when it reports failure, tear nothing down, replace no table and return a retryable error. Known exception: `PerfEvents::start()` still replaces `_events` without a gate (PROF-16359). `Profiler::startCpuEngine()` opens the - registration gate once any CPU engine started, so the gate is open iff it did; + registration gate once any CPU engine started, so the gate is open iff it did, and it + admits only the CPU engine. A wall engine that needs `registerThread()` needs its own + gate: opening this one would also admit a CPU engine that failed; - an engine's `stop()` must tolerate a concurrent `unregisterThread()`: `stop()` closes only the registration gate, and unregistration stays open while the engines stop; - close unregistration (`ThreadTablesFreeze`) only around an actual table replacement, diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 96ed3a7931..9002395b8a 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -285,9 +285,12 @@ int Profiler::registerThreadInEngines(int tid) { if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return -1; // not registered, as Engine::registerThread() reports it } + // The gate is open iff the CPU engine started (see startCpuEngine()), so it + // admits only that engine. No wall engine overrides + // Engine::registerThread(); one that needs per-thread registration needs a + // gate that tracks its own start, not this one. Engine *cpu = __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE); - Engine *wall = __atomic_load_n(&_instance->_wall_engine, __ATOMIC_ACQUIRE); - return cpu->registerThread(tid) | wall->registerThread(tid); + return cpu->registerThread(tid); } #ifdef UNIT_TEST static std::atomic g_test_last_unregistered_tid{-1}; @@ -1147,9 +1150,10 @@ void Profiler::disableEngines() { // a per-thread table (CTimer, PerfEvents) open the gate themselves before they // list threads, so opening it again here does nothing for them. For engines // without a table (ITimer, ITimerJvmti, J9) it opens the gate, so registration -// follows the profiler lifecycle rather than the engine choice. No wall engine -// overrides Engine::registerThread(), so a recording whose CPU engine did not -// start loses nothing with the gate closed. +// follows the profiler lifecycle rather than the engine choice. The gate admits +// only the CPU engine (registerThreadInEngines()): no wall engine overrides +// Engine::registerThread(), so a recording whose CPU engine did not start +// loses nothing with the gate closed. bool Profiler::startCpuEngine(Arguments &args) { Error error = _cpu_engine->start(args); if (error) { diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 01d0dbc667..39af8a086f 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -456,21 +456,22 @@ class alignas(alignof(SpinLock)) Profiler { void logStats(); void switchThreadEvents(jvmtiEventMode mode); /** - * Lets the native-thread hook's registerThread()/unregisterThread() reach - * the engines. A CPU engine with a per-thread table (CTimer, PerfEvents) + * Lets the native-thread hook's registerThread() reach the CPU engine, and + * reopens unregistration. A CPU engine with a per-thread table (CTimer, PerfEvents) * calls this from its start() once the table is sized and before it lists * the existing threads, so the hook handles a thread that starts or exits * while start() registers the existing ones. Profiler::startCpuEngine() * calls it again once the CPU engine's start() succeeds, and closes the * gate if it fails, so the gate is open iff the CPU engine started; - * wall-only, noop and failed-CPU recordings leave it closed. A future wall - * engine that needs registerThread() must open the gate itself. For CTimer + * wall-only, noop and failed-CPU recordings leave it closed. Because the + * gate tracks the CPU engine, registerThread() reaches only that engine. A + * wall engine that needs registerThread() needs its own gate; opening this + * one would also admit a CPU engine that failed. For CTimer * this misses no thread: the per-slot CAS resolves the hook and start() * registering one tid. PerfEvents::start()'s stale-fd sweep may close the * fd of a tid the hook registered after the thread list was taken, and * such a thread is then not sampled for the recording (PROF-16359). - * Engines without - * registerThread() never need the gate. Idempotent. + * Engines without registerThread() never need the gate. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 8edb635d22..dd24b81ca2 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -698,9 +698,8 @@ TEST(ProfilerEnginesGateTest, CpuEngineEnabledOnlyIfItStarted) { } // The idle fast path in registerThread() only short-cuts a closed gate: with -// the gate open the call still reaches the engine, and the in-flight count is -// back at zero afterwards either way. The return value is not checked with -// the gate open: the plain wall engine reports -1, which is OR-ed in. +// the gate open the call still reaches the engine and returns its result, and +// the in-flight count is back at zero afterwards either way. TEST(ProfilerEnginesGateTest, RegisterThreadFastPathDoesNotSkipOpenGate) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); @@ -713,13 +712,48 @@ TEST(ProfilerEnginesGateTest, RegisterThreadFastPathDoesNotSkipOpenGate) { EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); { ProfilerTestAccessor::RegistrationGate gate(profiler); - Profiler::registerThread(OS::threadId()); + EXPECT_EQ(0, Profiler::registerThread(OS::threadId())); EXPECT_EQ(1, cpu.register_calls.load()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); } +// A wall engine that counts registerThread() calls. +class CountingWallEngine : public Engine { +public: + std::atomic register_calls{0}; + + int registerThread(int tid) override { + register_calls.fetch_add(1); + return 0; + } +}; + +// The registration gate is open iff the CPU engine started, so it admits only +// the CPU engine. A wall engine never receives the hook through it: it would +// otherwise be called whenever the CPU engine started, even if the wall engine +// did not, and a wall engine that opened the gate itself would admit a CPU +// engine that failed. +TEST(ProfilerEnginesGateTest, RegistrationGateAdmitsOnlyCpuEngine) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + FakeCpuEngine cpu(/*fail=*/false, /*open_gate=*/false); + CountingWallEngine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + + int result; + { + ProfilerTestAccessor::RegistrationGate gate(profiler); + result = Profiler::registerThread(OS::threadId()); + } + + EXPECT_EQ(0, result); + EXPECT_EQ(1, cpu.register_calls.load()); + EXPECT_EQ(0, wall.register_calls.load()); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + // A negative tid must not index before CTimer's table. The page before the // table is inaccessible, so an access through _timers[-1] faults. TEST(CTimerTableTest, NegativeTidIsRejected) { diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 227d5ceb6c..9627147e8b 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -151,10 +151,13 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: table-less `ITimer`, `ITimerJvmti` and J9 engines) has started. It is cleared by `start()`, by `stop()`, by a failed CPU engine start and by `ThreadTablesFreeze`, so the gate is open iff the CPU engine started. - Wall-only, noop and failed-CPU recordings leave it closed; no wall engine - implements `registerThread()`, and one that ever does must open the gate - itself. While the gate is closed, `registerThread()` returns before it - touches the in-flight counter. + Wall-only, noop and failed-CPU recordings leave it closed. Because the + gate tracks the CPU engine, it admits only that engine: + `registerThread()` never calls the wall engine. No wall engine implements + `registerThread()`; one that ever does needs a gate of its own, because + opening this one would also admit a CPU engine that failed. While the + gate is closed, `registerThread()` returns before it touches the + in-flight counter. - `PerfEvents::start()`'s stale-fd sweep may close the fd of a tid the hook registered after the thread list was taken, and such a thread is then not sampled for the recording (PROF-16359). From 3a00a3394c5799b11f04135a12c0c01cbcdd35e1 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 16:37:07 -0400 Subject: [PATCH 23/26] PROF-16213: Keep check() off the engine slots; fix stale docs; test CTimer table Profiler::check() can run while a recording is RUNNING (checkState() rejects only NEW and ERROR), but it stored its selection into _cpu_engine, _wall_engine and _alloc_engine. The open registration gate then admitted an engine that never started, the wall signal handler cast wallEngine() to the wrong type, and stop() stopped the wrong engines. check() now selects into locals and never writes the slots. Docs and comments: - Replace profiler.cpp/ctimer_linux.cpp line references in ProfilingEngines.md, JfrIntegration.md and JvmSupportLayer.md with symbol references; this change had already shifted most of them. - The _wall_engine row no longer claims it receives registerThread. - List all six per-category native_mem_*_bytes families and drop the time-relative "ctimer is the newest". - A leaked in-flight count makes later gate closes time out, not wait forever. Tests: - TidAtBoundIsRejected: a tid equal to the bound never indexes past the timer table. The table is sized to the caller's own tid, so a wrongly accepted tid gets past timer_create() and faults on a guard page. - ResizeFreesOldTable: a resize frees the old table, checked through glibc mallinfo2() (skipped on musl and under ASan/TSan). Co-Authored-By: Claude Opus 5.5 --- ddprof-lib/src/main/cpp/nativeMem.h | 9 +-- ddprof-lib/src/main/cpp/profiler.cpp | 23 +++--- ddprof-lib/src/main/cpp/profiler.h | 7 +- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 79 +++++++++++++++++++++ doc/architecture/JfrIntegration.md | 15 ++-- doc/architecture/JvmSupportLayer.md | 14 ++-- doc/architecture/ProfilingEngines.md | 70 +++++++++--------- doc/reference/ProfilerMemoryRequirements.md | 10 +-- 8 files changed, 160 insertions(+), 67 deletions(-) diff --git a/ddprof-lib/src/main/cpp/nativeMem.h b/ddprof-lib/src/main/cpp/nativeMem.h index 3e4886cf37..c0253a7d5e 100644 --- a/ddprof-lib/src/main/cpp/nativeMem.h +++ b/ddprof-lib/src/main/cpp/nativeMem.h @@ -35,10 +35,11 @@ // DICTIONARY_ARENA_WASTE_BYTES remains a genuine nested waste figure for the // DICTIONARY arena. None of these nested counters are summed in here. // -// The exported counter names (native_mem_{live,avg,max,chunk_overhead}_bytes. -// ) derive from this table, and the set grows when a category is -// added (ctimer is the newest), so consumers should match native_mem_*_bytes. -// counters by prefix rather than hard-code the category list. +// The exported per-category counter names (native_mem_{live,avg,max, +// chunk_overhead,post_flush_live,post_flush_max}_bytes., written by +// Recording::writeNativeMem()) derive from this table, and the set grows when +// a category is added, so consumers should match native_mem_*_bytes. counters +// by prefix rather than hard-code the category list. #define DD_NATIVE_MEM_CATEGORY_TABLE(X) \ X(CALLTRACE, "calltrace") \ X(DICTIONARY, "dictionary") \ diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 9002395b8a..a688bee84c 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -248,7 +248,9 @@ bool Profiler::waitForHookCalls() { // in Log::warn()); acting on a cancellation there would unwind through // ~HookCallScope(), which aborts with the statically linked libgcc (see // run_with_cleanup() in libraryPatcher_linux.cpp), or skip it on musl and -// leave the in-flight count raised, so the next stop() would wait forever. +// leave the in-flight count raised. Every later closeThreadRegistration() and +// ThreadTablesFreeze would then time out, so stop() and every start() that +// replaces a table would fail. // // This is a manual disable/restore pair rather than a guard on purpose: with // asynchronous cancellation, current glibc's pthread_setcancelstate() acts on a @@ -321,7 +323,7 @@ void Profiler::unregisterThreadInEngines(int tid) { g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); return; #endif - // start() and check() may replace the engines concurrently. Either engine + // start() may replace the engines concurrently. Either engine // is safe to call: unregisterThread() only releases the exiting thread's own // slot, bound-checked against the engine's current table, and a CTimer table // being replaced is behind the gate above. PerfEvents replaces its table @@ -2159,17 +2161,20 @@ Error Profiler::check(Arguments &args) { error = checkJvmCapabilities(); + // check() may run while a recording is RUNNING (checkState() rejects only + // NEW and ERROR), so it selects into locals and leaves the engine slots to + // the engines that start() started: the hooks, the signal handlers (the wall + // handler casts wallEngine() to its own type) and stop() all use those. + Engine *cpu_engine = NULL; if (!error && (args._event != NULL || args._cpu >= 0)) { - __atomic_store_n(&_cpu_engine, selectCpuEngine(args), __ATOMIC_RELEASE); - error = _cpu_engine->check(args); + cpu_engine = selectCpuEngine(args); + error = cpu_engine->check(args); } if (!error && args._wall >= 0) { - __atomic_store_n(&_wall_engine, selectWallEngine(args), __ATOMIC_RELEASE); - error = _wall_engine->check(args); + error = selectWallEngine(args)->check(args); } if (!error && args._memory >= 0) { - _alloc_engine = selectAllocEngine(args); - error = _alloc_engine->check(args); + error = selectAllocEngine(args)->check(args); } if (!error && args._nativemem >= 0) { error = malloc_tracer.check(args); @@ -2180,7 +2185,7 @@ Error Profiler::check(Arguments &args) { if (!error) { if (args._cstack == CSTACK_DWARF && !DWARF_SUPPORTED) { return Error("DWARF unwinding is not supported on this platform"); - } else if (args._cstack == CSTACK_LBR && _cpu_engine != &perf_events) { + } else if (args._cstack == CSTACK_LBR && cpu_engine != &perf_events) { return Error("Branch stack is supported only with PMU events"); } else if (_cstack >= CSTACK_VM && !(VMStructs::hasStackStructs() && OS::isLinux())) { return Error( diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 39af8a086f..6aed0cde0f 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -150,9 +150,10 @@ class alignas(alignof(SpinLock)) Profiler { ThreadFilter _thread_filter; CallTraceStorage _call_trace_storage; FlightRecorder _jfr; - // Replaced by start() and check() while thread unregistration is open, so - // the hook paths (registerThreadInEngines(), unregisterThreadInEngines()) - // load them with acquire and those writers store them with release. + // Replaced only by start(), while thread unregistration is open, so the + // hook paths (registerThreadInEngines(), unregisterThreadInEngines()) load + // them with acquire and start() stores them with release. check() selects + // into locals: it may run during a recording. Engine *_cpu_engine; Engine *_wall_engine = NULL; Engine *_alloc_engine; diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index dd24b81ca2..011ea254ca 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -19,11 +19,21 @@ #include #include #include +#include #include #include #include #include +// mallinfo2() reports the bytes in use only with glibc's own allocator: it is +// glibc 2.33+ only, and the sanitizer allocators report zeros. Nested so that +// __GLIBC_PREREQ is expanded only where it is defined (not on musl). +#if defined(__GLIBC__) && !defined(ASAN_ENABLED) && !defined(TSAN_ENABLED) +#if __GLIBC_PREREQ(2, 33) +#define HAS_GLIBC_MALLINFO2 1 +#endif +#endif + // The failure-path tests force calloc to fail with a negative count, which // makes count * size overflow. The CI sanitizer jobs run gtest binaries // without *SAN_OPTIONS, so allocator_may_return_null defaults to 0 and the @@ -185,6 +195,21 @@ class CTimerTestAccessor { CTimer::_max_timers = max_timers; } + // Sets CTimer's timer signal for its lifetime and restores the previous one. + // It is 0 until a CTimer has started, and timer_create() rejects signal 0. + class SignalSwap { + public: + explicit SignalSwap(int signal) : _signal(CTimer::_signal) { + CTimer::_signal = signal; + } + ~SignalSwap() { CTimer::_signal = _signal; } + SignalSwap(const SignalSwap &) = delete; + SignalSwap &operator=(const SignalSwap &) = delete; + + private: + int _signal; + }; + // Installs a caller-owned table for its lifetime and puts the previous one // back on destruction, without freeing either. class TimerTableSwap { @@ -394,6 +419,31 @@ TEST_F(CTimerNativeMemTest, ResizeReplacesTableWhenHookCallsDrain) { EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); } +// A resize frees the old table, which the NM_CTIMER gauge alone cannot show: +// it is decremented whether or not the old table is freed. glibc's +// mallinfo2() reports the bytes the allocator has handed out (heap plus mmap +// chunks), so replacing an N1-slot table with an N2-slot one raises that by +// about (N2 - N1) slots when the old table is freed, and by N2 slots when it +// leaks. The sanitizer allocators and musl do not report through mallinfo2(). +TEST_F(CTimerNativeMemTest, ResizeFreesOldTable) { +#ifndef HAS_GLIBC_MALLINFO2 + GTEST_SKIP() << "needs glibc mallinfo2() without a sanitizer allocator"; +#else + const int old_slots = 1 << 16; + const int new_slots = 1 << 17; + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(old_slots)); + struct mallinfo2 before = mallinfo2(); + ASSERT_TRUE(CTimerTestAccessor::resizeTimers(new_slots)); + struct mallinfo2 after = mallinfo2(); + + long long in_use_before = (long long)(before.uordblks + before.hblkhd); + long long in_use_after = (long long)(after.uordblks + after.hblkhd); + long long grown = in_use_after - in_use_before; + // Halfway between the freed (new - old) and leaked (new) outcomes. + EXPECT_LT(grown, bytes(new_slots) - bytes(old_slots) / 2); +#endif +} + // Profiler::registerThread() is reached from the pthread hook without // _state_lock. Until start() has sized the engines' per-thread structures // (CTimer::resizeTimers()) and set the engines-ready flag, it must return @@ -777,6 +827,35 @@ TEST(CTimerTableTest, NegativeTidIsRejected) { munmap(base, 2 * page_size); } +// A tid equal to the bound must not index one past CTimer's table. The table +// is sized to the calling thread's own tid, so that tid is one past its end, +// and the page after the table is inaccessible. The tid is live and the timer +// signal is valid, so a registerThread() that wrongly passed the bound would +// get past timer_create() and fault on the CAS into the guard page. +TEST(CTimerTableTest, TidAtBoundIsRejected) { + size_t page_size = (size_t)sysconf(_SC_PAGESIZE); + int tid = OS::threadId(); + ASSERT_GT(tid, 0); + size_t table_bytes = (size_t)tid * sizeof(int); + size_t table_pages = (table_bytes + page_size - 1) / page_size; + size_t map_bytes = (table_pages + 1) * page_size; + char *base = (char *)mmap(nullptr, map_bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ASSERT_NE(MAP_FAILED, (void *)base); + char *guard = base + table_pages * page_size; + ASSERT_EQ(0, mprotect(guard, page_size, PROT_NONE)); + int *table = (int *)(guard - table_bytes); + { + CTimerTestAccessor::TimerTableSwap swap(table, tid); + CTimerTestAccessor::SignalSwap signal(SIGPROF); + CTimer engine; + CTimerTestAccessor::unregisterThread(engine, tid); + EXPECT_EQ(-1, CTimerTestAccessor::registerThread(engine, tid)); + EXPECT_EQ(0, table[tid - 1]); + } + munmap(base, map_bytes); +} + // Thread body for the cancellation test: makes a hook call with asynchronous // cancellation enabled, then reaches a cancellation point. A plain C-style // thread is used so that no C++ frame with cleanups sits above the call when diff --git a/doc/architecture/JfrIntegration.md b/doc/architecture/JfrIntegration.md index 402c5a0158..f25275b993 100644 --- a/doc/architecture/JfrIntegration.md +++ b/doc/architecture/JfrIntegration.md @@ -62,10 +62,10 @@ event silently. (`flightRecorder.h:84`, `283`). They are indexed by the profiler's lock stripe, not by thread: -1. `Profiler::getLockIndex(tid)` hashes the thread ID to a stripe - (`profiler.cpp:331-336`). +1. `Profiler::getLockIndex(tid)` hashes the thread ID to a stripe. 2. The caller `tryLock()`s that stripe, then up to two others; if all three are - taken the sample is dropped (`profiler.cpp:781-784`). + taken the sample is dropped (`Profiler::recordSample()` and the other + `record*()` methods). 3. With the stripe lock held it calls `FlightRecorder::recordEvent(lock_index, ...)`, which writes into `_buf[lock_index]` (`flightRecorder.cpp:2866-2907`). @@ -138,7 +138,8 @@ events write the context snapshot taken when the allocation was tracked - `wallClockEpoch()`, `recordTraceRoot()`, `recordQueueTime()`, `recordDatadogSetting()`, `recordHeapUsage()` and the reference-chain writers use `_buf[lock_index]`; their `Profiler` callers take the stripe lock first - (`profiler.cpp:890-996`, `1972-1984`). + (`Profiler::writeDatadogProfilerSetting()`, `writeHeapUsage()`, and the + reference-chain drains in `dump()` and `stop()`). - `recordLog()` builds the event in an `alloca` buffer and writes it to the file at once, without a stripe lock (`flightRecorder.cpp:2945-2966`). - `cpuMonitorCycle()` writes `jdk.CPULoad` into `_cpu_monitor_buf`, which @@ -195,10 +196,10 @@ properties and native libraries can be turned off with `NO_SYSTEM_INFO`, ### How `Profiler` drives it -`Profiler::dump()` (`profiler.cpp:2142-2214`) first emits pending +`Profiler::dump()` first emits pending reference-chain events under a stripe lock, refreshes libraries and thread names, then runs `_jfr.dump()` inside `rotateDictsAndRun()`. `Profiler::stop()` -runs `_jfr.stop()` the same way (`profiler.cpp:2064`). +runs `_jfr.stop()` the same way. `rotateDictsAndRun()` (`profiler.h:245-268`): @@ -257,7 +258,7 @@ static string table filled by `getId()` (`jfrMetadata.h:114-122`); `Profiler::start()` calls `JfrMetadata::reset()` then `initialize()` with `args._context_attributes`, truncated to `DD_TAGS_CAPACITY`, before -`_jfr.start()` (`profiler.cpp:1760-1768`). `reset()` deletes the tree and +`_jfr.start()`. `reset()` deletes the tree and re-registers `"root"` as string 0 so the root element's name stays valid (`jfrMetadata.cpp:43-60`). Both require `Profiler::_state_lock` and must not run while engines are active (`jfrMetadata.h:276-284`). diff --git a/doc/architecture/JvmSupportLayer.md b/doc/architecture/JvmSupportLayer.md index 0378fc3830..583786ea43 100644 --- a/doc/architecture/JvmSupportLayer.md +++ b/doc/architecture/JvmSupportLayer.md @@ -83,11 +83,11 @@ predicates. More detection-dependent setup: - `can_retransform_any_class` is disabled on J9 (`vmEntry.cpp:502`). - The JFR `RequestStackTrace` extension is probed only on HotSpot (`vmEntry.cpp:519-521`). - The profiler installs its SIGSEGV/SIGBUS handlers only on HotSpot and J9 - (`profiler.cpp:1270`). -- On Zing, `GC_TRACES | LAST_JAVA_PC` is forced into `_safe_mode` (`profiler.cpp:1689`). + (`Profiler::setupSignalHandlers()`). +- On Zing, `GC_TRACES | LAST_JAVA_PC` is forced into `_safe_mode` (`Profiler::start()`). `JVMSupport::initialize()` (`jvmSupport.cpp:32-57`) runs after detection, from -`Profiler::start()`/`Profiler::init()` (`profiler.cpp:1519`, `profiler.cpp:1539`). +`Profiler::start()`/`Profiler::init()`. It calls `JVMThread::initialize()`, then checks the `ProfiledThread` TLS key. If either step fails, profiling is disabled. @@ -116,7 +116,7 @@ gets HotSpot declarations, but it should still call only `JVMSupport::*`. Callers in shared code: -- Stack walking calls `walkJavaStack` (`profiler.cpp:819`). +- Stack walking calls `walkJavaStack` (`Profiler::recordSample()`). - `stackWalker.cpp:80-82` and `perfEvents_linux.cpp` call `isJitCode` and `canUnwind`. Both are signal-handler paths, and on non-HotSpot VMs they reduce to constant returns. @@ -183,7 +183,7 @@ Caveats: (for example, a build without JFR), `initLazyResolution()` fails, and `initExecution()` falls back to `Partial_loaded` (`jvmSupport.cpp:86-91`). - **`cstack=vm` needs `VMStructs`.** If `VMStructs::hasStackStructs()` is false, - `cstack=vm` is downgraded to DWARF or `no` (`profiler.cpp:1728-1732`). + `cstack=vm` is downgraded to DWARF or `no` (`Profiler::start()`). ### `j9/` @@ -196,7 +196,7 @@ Caveats: - **The ASGCT-or-JVMTI choice is made per engine.** `Profiler::selectCpuEngine` and `selectWallEngine` choose `J9WallClock` (the JVMTI path) when any of these - holds (`profiler.cpp:1376-1387`, `profiler.cpp:1423-1435`): + holds: - `J9Support::shouldUseAsgct()` is false. The system property `dd.profiling.ddprof.j9.sampler` set to `jvmti` makes it false (`j9/j9Support.h:108-126`). @@ -223,7 +223,7 @@ behavior is shared code that branches on `VM::isZing()`: - Java stacks are collected with ASGCT (`jvmSupport.cpp:112-117`). - Runtime entries are marked (`vmEntry.cpp:378-379`). -- `_safe_mode` is forced (`profiler.cpp:1689`). +- `_safe_mode` is forced (`Profiler::start()`). Caveat: `JVMThread::nativeThreadId()` sends Zing to `VMThread::nativeThreadId()`. That function returns -1 unless diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 9627147e8b..2a8eef3061 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -46,7 +46,7 @@ the header and no `CpuEngine::` member is defined anywhere. `PerfEvents`, ### Engine slots in `Profiler` -`profiler.cpp:73-82` declares each engine as a file-static singleton: +The top of `profiler.cpp` declares each engine as a file-static singleton: `noop_engine` (a bare `Engine`, used when a mode is off), `malloc_tracer`, `perf_events`, `wall_asgct_engine`, `wall_jvmti_engine`, `j9_engine`, `itimer`, `itimer_jvmti`, `ctimer`, and `ctimer_jvmti`. `ObjectSampler` is a @@ -55,9 +55,9 @@ separate singleton (`ObjectSampler::instance()`, | Slot | Chosen by | Notes | |------|-----------|-------| -| `_cpu_engine` | `selectCpuEngine()` (`profiler.cpp:1371`) | Receives `registerThread`/`unregisterThread` and `enableEvents` | -| `_wall_engine` | `selectWallEngine()` (`profiler.cpp:1418`) | Same as above | -| `_alloc_engine` | `selectAllocEngine()` (`profiler.cpp:1453`) | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | +| `_cpu_engine` | `selectCpuEngine()` | Receives `registerThread`/`unregisterThread` and `enableEvents` | +| `_wall_engine` | `selectWallEngine()` | Receives `unregisterThread` and `enableEvents`, not `registerThread`: the registration gate admits only the CPU engine (see Thread registration) | +| `_alloc_engine` | `selectAllocEngine()` | `ObjectSampler` if `VM::canSampleObjects()`, else `noop_engine` | `MallocTracer` has no slot. `Profiler::start()`/`stop()` call `malloc_tracer` directly. `LivenessTracker` and `NativeMem` are not `Engine`s. @@ -65,7 +65,7 @@ directly. `LivenessTracker` and `NativeMem` are not `Engine`s. ## Lifecycle ``` -Profiler::start() profiler.cpp:1555 +Profiler::start() _engines_ready cleared registration closed until the CPU engine starts; unregistration stays open selectCpuEngine / selectWallEngine engine pointers stored with release @@ -100,13 +100,13 @@ Profiler::start() profiler.cpp:1555 - A failing CPU, wall, or alloc `start()` is logged and the remaining modes still start. Start fails only if no mode activated. -`Profiler::stop()` (`profiler.cpp:1922`) runs in reverse: +`Profiler::stop()` runs in reverse: 1. `disableEngines()` calls `enableEvents(false)` on the CPU and wall engines - only (`profiler.cpp:1106`). + only. 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error - and tears nothing down (`profiler.cpp:1935`). + and tears nothing down. 3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook call that passed the check is still inside an engine. No `registerThread()` call (pthread hook or JVMTI `ThreadStart`) can then arm @@ -125,8 +125,10 @@ Profiler::start() profiler.cpp:1555 `malloc_tracer.stop()`, the socket sampler, `_wall_engine->stop()`, `_cpu_engine->stop()`. -`Profiler::check()` (`profiler.cpp:2075`) runs the same selection and calls -each engine's `check()` without starting anything. +`Profiler::check()` runs the same selection and calls each engine's `check()` +without starting anything. It selects into locals and never replaces the +engine slots: it may run while a recording is `RUNNING`, and the hooks, the +signal handlers and `stop()` use the slots of the engines `start()` started. ### Thread registration @@ -171,10 +173,10 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: threads, its slot stays occupied after it dies (PROF-16359). The gate stays set while idle, because `PerfEvents::stop()` keeps per-thread fds open and this call closes the fd of a thread that exits after `stop()`. - - `start()` and `check()` replace `_cpu_engine`/`_wall_engine` while - unregistration is open, so they store the pointers with release and the - hook paths load them with acquire. An unregistration that reaches the - previous engine only releases that thread's own slot. + - `start()` replaces `_cpu_engine`/`_wall_engine` while unregistration is + open, so it stores the pointers with release and the hook paths load them + with acquire. An unregistration that reaches the previous engine only + releases that thread's own slot. `check()` never writes the slots. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` (`perfEvents_linux.cpp:980`). @@ -220,7 +222,8 @@ Profiler::recordSample[Delegated]() `SIGNAL_HANDLER_GUARD_OR_DROP` is defined at `guards.h:115`. Inside `recordSample()`, the handler tries to take one of `CONCURRENCY_LEVEL` (16) striped spin locks with `tryLock()`, three attempts at most. If all three fail, -the sample is dropped and counted as `ticks_skipped` (`profiler.cpp:782-793`). +the sample is dropped and counted as `ticks_skipped` (`Profiler::recordSample()` and the other +`record*()` methods). No handler blocks. The `*Jvmti` variants call `recordSampleDelegated()` instead, which asks @@ -231,7 +234,7 @@ HotSpot's JFR `RequestStackTrace` JVMTI extension to walk the stack ### Selection -`selectCpuEngine()` (`profiler.cpp:1371-1416`): +`Profiler::selectCpuEngine()`: ``` cpu not requested and event is NULL/"noop" -> noop_engine @@ -267,25 +270,27 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Mechanism**: `registerThread(tid)` creates one POSIX timer per thread with the raw `__NR_timer_create` syscall. The clock is `((~tid) << 3) | 6`, the thread's per-thread scheduler CPU clock - (`ctimer_linux.cpp:46-48`). The raw syscall is used because the libc wrapper - accepts only predefined clocks (`ctimer_linux.cpp:86`). The timer notifies + (`thread_cpu_clock()` in `ctimer_linux.cpp`). The raw syscall is used because the libc wrapper + accepts only predefined clocks. The timer notifies with `SIGEV_THREAD_ID` targeting `tid`, carrying - `sival_ptr = SignalCookie::cpu()` (`ctimer_linux.cpp:73-84`). It is armed + `sival_ptr = SignalCookie::cpu()` (`CTimer::registerThread()`). It is armed periodic with `timer_settime`. -- **Signal**: `SIGPROF` (`ctimer_linux.cpp:181`). +- **Signal**: `SIGPROF` (set in `CTimer::startWithTableSize()`). - **Origin check**: the handler accepts only `si_code == SI_TIMER` carrying the CPU cookie. Anything else is forwarded with `OS::forwardForeignSignal()` - (`ctimer_linux.cpp:292-296`). + (`CTimer::signalHandler()`). - **Per-thread table**: `_timers[tid]` holds `timer_id + 1` (0 means empty), sized from `OS::getMaxThreadId()`. Registration and unregistration claim slots with CAS, so concurrent register/unregister of one tid deletes the - timer exactly once (`ctimer_linux.cpp:94-132`). A tid at or above + timer exactly once (`CTimer::registerThread()`, `unregisterThread()`). A tid + at or above `_max_timers` logs a warning and is not profiled. The table is accounted under `NM_CTIMER` at its full allocated size (see Profiler Self-Accounting). -- **`stop()`** unregisters every slot (`ctimer_linux.cpp:212-218`). +- **`stop()`** unregisters every slot. - **`CTimerJvmti`**: runs `CTimer::start()` and then replaces the `SIGPROF` handler with one that calls `recordSampleDelegated(ucontext, ...)` - (`ctimer_linux.cpp:227-283`). It has no ASGCT fallback. + (`CTimerJvmti::start()`, `CTimerJvmti::signalHandler()`). It has no ASGCT + fallback. ### `PerfEvents` (`perfEvents_linux.cpp`, Linux only) @@ -317,7 +322,7 @@ resolved through `Arguments::cpuSamplerInterval()`. does not exclude thread unregistration or the signal-path readers (`walkKernel()`, `resetBuffer()`) while it does so (PROF-16359). - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this - engine (`profiler.cpp:1725`, `1745`). + engine (`Profiler::start()`, `Profiler::getNativeTrace()`). ### `ITimer` / `ITimerJvmti` (`itimer.cpp`, all platforms) @@ -352,7 +357,7 @@ dedicated profiler pthread picks threads and signals each one directly. ### Selection -`selectWallEngine()` (`profiler.cpp:1418-1451`): on OpenJ9, either +`Profiler::selectWallEngine()`: on OpenJ9, either `J9WallClock` (JVMTI polling) or `WallClockASGCT`. On HotSpot with `jvmtistacks` and `VM::canRequestStackTrace()`, `WallClockJvmti`. Otherwise, `WallClockASGCT`; a requested `JVMTI` sampler that is not available falls back @@ -423,7 +428,7 @@ running. The source tree has no TLAB-specific engine class. Heap allocation sampling is JVMTI `SampledObjectAlloc`. `BCI_ALLOC_OUTSIDE_TLAB` exists in `vmEntry.h:27` -and is handled in `profiler.cpp:430` and `hotspot/hotspotSupport.cpp:123`, but no +and is handled in `Profiler::getNativeTrace()` and `hotspot/hotspotSupport.cpp:123`, but no engine produces it; `ObjectSampler` always records `BCI_ALLOC`. - **Capability**: `VM::canSampleObjects()` requires @@ -492,7 +497,7 @@ chains are described in survivors. `cleanup_table()` runs from `flush()`, `stop()`, the overflow path in `track()`, and `maybeForceCleanup()` (called from the reference-chain thread). -- **Output**: `Profiler::dump()` calls `flush()` (`profiler.cpp:2153`). The +- **Output**: `Profiler::dump()` calls `flush()`. The flush runs a cleanup and then, under the exclusive lock, emits one `BCI_LIVENESS` deferred sample per surviving entry through `recordDeferredSample()` (`livenessTracker.cpp:1962`). When heap-usage @@ -574,10 +579,11 @@ pages holding live tids are ever touched. With a `pid_max` of 4194304 the bounds. Only `NM_PERF`'s ring pages are resident. Each category is exported per chunk as -`native_mem_{live,avg,max,chunk_overhead}_bytes.`, with names taken -from `DD_NATIVE_MEM_CATEGORY_TABLE`. The set grows when categories are added -(`ctimer` is the newest), so consumers should match these counters by prefix -rather than hard-code the category list. +`native_mem_{live,avg,max,chunk_overhead,post_flush_live,post_flush_max}_bytes.` +(`Recording::writeNativeMem()`), with names taken from +`DD_NATIVE_MEM_CATEGORY_TABLE`. The set grows when categories are added, so +consumers should match these counters by prefix rather than hard-code the +category list. ## Counters by Engine diff --git a/doc/reference/ProfilerMemoryRequirements.md b/doc/reference/ProfilerMemoryRequirements.md index 81ee6eec68..9dd378d795 100644 --- a/doc/reference/ProfilerMemoryRequirements.md +++ b/doc/reference/ProfilerMemoryRequirements.md @@ -8,11 +8,11 @@ The sizes below are estimates. The profiler's own native allocations are measure runtime: `NativeMem` (`ddprof-lib/src/main/cpp/nativeMem.h`) keeps live, average and peak bytes for each category (`calltrace`, `dictionary`, `line_tables`, `thread_local`, `jfr_buffers`, `method_map`, `perf`, `ctimer`, ...), and every chunk carries them as -`datadog.ProfilerCounter` events named `native_mem_live_bytes.`, -`native_mem_avg_bytes.`, `native_mem_max_bytes.` and -`native_mem_chunk_overhead_bytes.`. jmethodIDs are allocated by the JVM and are -not part of these figures. The category set is not fixed (`ctimer` is the newest), so -match these counters by prefix rather than by a fixed list. The `ctimer` gauge and the +`datadog.ProfilerCounter` events named +`native_mem_{live,avg,max,chunk_overhead,post_flush_live,post_flush_max}_bytes.` +(`Recording::writeNativeMem()`). jmethodIDs are allocated by the JVM and are +not part of these figures. The category set is not fixed, so match these counters by +prefix rather than by a fixed list. The `ctimer` gauge and the table part of the `perf` gauge count reserved, mostly untouched tid-indexed tables (`pid_max` slots each), so they overstate RSS and are upper bounds. From ad06a065926e291d47cb08dfb8489319f7dbc959 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 19:20:29 -0400 Subject: [PATCH 24/26] PROF-16213: Harden engine changes, gate timeouts and PerfEvents table loss Engine change: stop() leaves PerfEvents' per-thread fds open so that unregisterThread() closes them as threads exit, but once start() selected another CPU engine those exits went to the new engine and the fds leaked. Add Engine::releaseThreads() (PerfEvents closes every fd and ring) and Profiler::replaceCpuEngine(), which has the previous engine release its threads under a ThreadTablesFreeze before the slot changes. PerfEvents also releases its threads before start() frees the table on a pid_max change. PerfEvents allocation failure: reset _max_events to -1, log an error and fail start() before the signal handler, the hook and the registration gate are set up. walkKernel() and resetBuffer() skip a missing table: fds armed by an earlier recording keep delivering SIGPROF. Gate timeouts: count registration and unregistration calls separately. closeThreadRegistration() waits only for registration calls, because unregistration stays open during stop() and its count may not reach zero under thread churn. ThreadTablesFreeze waits for both. If closeThreadRegistration() times out, the profiler goes to ERROR, which rejects every later start(), the only place a table is replaced: stop() still completes, since it frees no table, and start() rolls back its setup. The rollback is one helper, rollBackStart(), which now also restores the library patches. Also: the gate contract is stated once, on the gate fields in profiler.h, with pointers elsewhere; StartOpensThreadRegistration installs its CTimer as the CPU engine while the gate is open. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 15 +- ddprof-lib/src/main/cpp/ctimer_linux.cpp | 8 +- ddprof-lib/src/main/cpp/engine.h | 5 + ddprof-lib/src/main/cpp/perfEvents.h | 1 + ddprof-lib/src/main/cpp/perfEvents_linux.cpp | 43 +++- ddprof-lib/src/main/cpp/profiler.cpp | 186 ++++++++++------- ddprof-lib/src/main/cpp/profiler.h | 114 ++++++----- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 204 +++++++++++++++++-- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 11 + doc/architecture/ProfilingEngines.md | 61 ++++-- 10 files changed, 475 insertions(+), 173 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 895e62996e..5a5f072b4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -284,16 +284,23 @@ for a temporary window. `Profiler::openThreadRegistration()`, then lists the existing threads. Free or replace a table only while its gate is closed: closing a gate (`closeThreadRegistration()`, `ThreadTablesFreeze`) waits until no call is inside an - engine. The wait is bounded; when it reports failure, tear nothing down, replace no - table and return a retryable error. Known exception: `PerfEvents::start()` still + engine. Each gate counts its calls separately: `closeThreadRegistration()` waits only + for registration calls, because unregistration stays open during `stop()`; + `ThreadTablesFreeze` waits for both. The wait is bounded; when it reports failure, + free or replace no table. A `ThreadTablesFreeze` timeout fails the operation; a + `closeThreadRegistration()` timeout puts the profiler in `ERROR` (`stop()` still + completes, since it frees no table), so no later `start()` replaces a table under + the stuck call. Engine + resources kept across `stop()` are released (`Engine::releaseThreads()`) when another + engine replaces the CPU engine. Known exception: `PerfEvents::start()` still replaces `_events` without a gate (PROF-16359). `Profiler::startCpuEngine()` opens the registration gate once any CPU engine started, so the gate is open iff it did, and it admits only the CPU engine. A wall engine that needs `registerThread()` needs its own gate: opening this one would also admit a CPU engine that failed; - an engine's `stop()` must tolerate a concurrent `unregisterThread()`: `stop()` closes only the registration gate, and unregistration stays open while the engines stop; -- close unregistration (`ThreadTablesFreeze`) only around an actual table replacement, - never for a whole `start()`: an exit dropped while it is closed leaves a stale slot +- close unregistration (`ThreadTablesFreeze`) only around an actual table replacement + or a CPU engine change, never for a whole `start()`: an exit dropped while it is closed leaves a stale slot that a thread reusing the tid cannot claim, and `start()`/`stop()` cycles repeat; - restore the cancellation state outside every destructor-bearing scope, never from a destructor: current glibc acts on a pending asynchronous cancellation as soon as it diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp index 886d62284f..ea7e1e3784 100644 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ b/ddprof-lib/src/main/cpp/ctimer_linux.cpp @@ -159,15 +159,17 @@ bool CTimer::resizeTimers(int max_timers) { // A restart with an unchanged pid_max keeps the table and never closes thread // unregistration, so an exit during start() still releases its timer. Only an -// actual replacement freezes the hook calls; if one is still inside the table -// after the wait budget, start() fails retryably and keeps the old table. +// actual replacement freezes the hook calls. If one is still inside the table +// after the wait budget, start() fails and keeps the old table; whether a +// later start() can succeed depends on the stuck call (see +// Profiler::startCpuEngine()). Error CTimer::sizeTimerTable(int max_timers) { if (max_timers == _max_timers) { return Error::OK; } Profiler::ThreadTablesFreeze freeze(Profiler::instance()); if (!freeze.drained()) { - return Error("thread hook calls did not drain; retry start()"); + return Error("thread hook calls did not drain"); } if (!resizeTimers(max_timers)) { return Error("Failed to allocate CPU timer table"); diff --git a/ddprof-lib/src/main/cpp/engine.h b/ddprof-lib/src/main/cpp/engine.h index c9e75e2986..2deffca625 100644 --- a/ddprof-lib/src/main/cpp/engine.h +++ b/ddprof-lib/src/main/cpp/engine.h @@ -1,5 +1,6 @@ /* * Copyright 2017 Andrei Pangin + * Copyright 2026, Datadog, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -53,6 +54,10 @@ class Engine { virtual int registerThread(int tid) { return -1; } virtual void unregisterThread(int tid) {} + // Releases the per-thread resources the engine keeps across stop(). Called + // when another engine replaces it in its Profiler slot: thread exits no + // longer reach it after that, so nothing else would release them. + virtual void releaseThreads() {} virtual void enableEvents(bool enabled) { // do nothing diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h index 560d214352..6f9eb32a72 100644 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ b/ddprof-lib/src/main/cpp/perfEvents.h @@ -52,6 +52,7 @@ class PerfEvents : public Engine { virtual int registerThread(int tid); virtual void unregisterThread(int tid); + void releaseThreads(); long interval() const { return _interval; } const char *name() { return "PerfEvents"; } diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp index 4de0f227e0..842e57db00 100644 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp @@ -962,16 +962,29 @@ Error PerfEvents::start(Arguments &args) { // sites); capture the old size first since _max_events is overwritten below. long long old_bytes = (_events != NULL) ? (long long)((size_t)_max_events * sizeof(PerfEvent)) : 0; + // The table is the only record of which fd and ring belong to which tid + // (stop() keeps them). Release them while it still exists; otherwise + // they leak, and stay armed, once it is freed. + releaseThreads(); free(_events); if (old_bytes > 0) { NativeMem::record(NM_PERF, -old_bytes); } _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); - _max_events = max_events; - if (_events != NULL) { - NativeMem::record(NM_PERF, - (long long)((size_t)max_events * sizeof(PerfEvent))); + if (_events == NULL) { + // Back to the not-started value, so registerThread() and + // unregisterThread() reject every tid. Return before the signal + // handler, the hook and the registration gate are set up: start() + // fails, so the CPU engine is neither registered nor enabled for this + // recording. The next start() retries the allocation. + _max_events = -1; + Log::error("perfEvents: could not allocate the table for %d threads; " + "PerfEvents is disabled for this recording", max_events); + return Error("Failed to allocate perf event table"); } + _max_events = max_events; + NativeMem::record(NM_PERF, + (long long)((size_t)max_events * sizeof(PerfEvent))); } OS::installSignalHandler(SIGPROF, signalHandler); @@ -1056,12 +1069,28 @@ void PerfEvents::stop() { // check whether the thread has been registered already on start. } +// stop() keeps every fd and ring, and only unregisterThread() releases them +// when a thread exits. Releases them all when nothing else could: +// Profiler::replaceCpuEngine() calls this under a ThreadTablesFreeze once +// another CPU engine replaces this one, and start() calls it before it frees +// the table on a pid_max change. The start() call runs without a freeze, so a +// concurrent unregisterThread() of the same tid is still possible there +// (PROF-16359). +void PerfEvents::releaseThreads() { + for (int tid = 0; tid < _max_events; tid++) { + unregisterThread(tid); + } +} + int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, StackContext *java_ctx) { if (!(_ring & RING_KERNEL)) { // we are not capturing kernel stacktraces return 0; } + if (_events == NULL || tid < 0 || tid >= _max_events) { + return 0; // no table: the allocation failed (see start()) + } PerfEvent *event = &_events[tid]; if (!event->tryLock()) { @@ -1152,6 +1181,12 @@ int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, } void PerfEvents::resetBuffer(int tid) { + // Fds left armed by an earlier recording keep delivering SIGPROF, and the + // handler calls this on every exit, also after a failed table allocation + // left _events NULL (_max_events -1). + if (_events == NULL || tid < 0 || tid >= _max_events) { + return; + } PerfEvent *event = &_events[tid]; if (!event->tryLock()) { return; // the event is being destroyed diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index a688bee84c..3da5b59f3e 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -174,7 +174,9 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // Marks a native-thread hook call as in flight for the duration of a // Profiler::registerThread()/unregisterThread() call, so a gate can be closed -// and the calls that already passed it waited for. The increment and the gate +// and the calls that already passed it waited for. Each gate has its own +// counter (_register_calls_inflight, _unregister_calls_inflight). The +// increment and the gate // load in the caller are seq_cst, pairing with the seq_cst gate store and // counter load on the closing side: either the hook sees the gate closed, or // the closer sees the call in flight and waits for it. registerThreadInEngines() @@ -200,16 +202,23 @@ void Profiler::openThreadRegistration() { bool Profiler::closeThreadRegistration() { _engines_ready.store(false, std::memory_order_seq_cst); - return waitForHookCalls(); + return waitForHookCalls(/*include_unregistration=*/false); } -bool Profiler::waitForHookCalls() { +bool Profiler::waitForHookCalls(bool include_unregistration) { // Hook calls only do a few syscalls per engine, so the wait is short. It is // bounded so that a call that never leaves (or a leaked in-flight count) - // fails start()/stop() with a retryable error instead of wedging them under - // _state_lock. clock_gettime() rather than OS::nanotime(): see - // SamplerPerf::primeClock() in start(). - if (_hook_calls_inflight.load(std::memory_order_seq_cst) == 0) { + // fails the caller instead of wedging it under _state_lock. + // clock_gettime() rather than OS::nanotime(): see SamplerPerf::primeClock() + // in start(). + auto inflight = [this, include_unregistration]() { + int calls = _register_calls_inflight.load(std::memory_order_seq_cst); + if (include_unregistration) { + calls += _unregister_calls_inflight.load(std::memory_order_seq_cst); + } + return calls; + }; + if (inflight() == 0) { return true; } long long budget = _hook_drain_timeout_ns; @@ -224,7 +233,7 @@ bool Profiler::waitForHookCalls() { deadline.tv_sec += (time_t)(nsec / 1000000000LL); deadline.tv_nsec = (long)(nsec % 1000000000LL); - while (_hook_calls_inflight.load(std::memory_order_seq_cst) != 0) { + while (inflight() != 0) { struct timespec now; if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) { Log::warn("Profiler: clock_gettime(CLOCK_MONOTONIC) failed (errno=%d) " @@ -233,7 +242,7 @@ bool Profiler::waitForHookCalls() { } if (now.tv_sec > deadline.tv_sec || (now.tv_sec == deadline.tv_sec && now.tv_nsec >= deadline.tv_nsec)) { - int remaining = _hook_calls_inflight.load(std::memory_order_seq_cst); + int remaining = inflight(); Log::warn("Profiler: timed out after %ldms waiting for %d in-flight " "thread hook call(s)", (long)(budget / 1000000LL), remaining); return false; @@ -281,16 +290,11 @@ int Profiler::registerThreadInEngines(int tid) { if (!_instance->_engines_ready.load(std::memory_order_acquire)) { return -1; // not registered, as Engine::registerThread() reports it } - HookCallScope scope(_instance->_hook_calls_inflight); - // Threads created before the gate opens are picked up by the CPU engine's - // start(), which opens the gate before it lists the existing threads. + HookCallScope scope(_instance->_register_calls_inflight); if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return -1; // not registered, as Engine::registerThread() reports it } - // The gate is open iff the CPU engine started (see startCpuEngine()), so it - // admits only that engine. No wall engine overrides - // Engine::registerThread(); one that needs per-thread registration needs a - // gate that tracks its own start, not this one. + // The gate admits only the CPU engine (see _engines_ready). Engine *cpu = __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE); return cpu->registerThread(tid); } @@ -306,10 +310,8 @@ void Profiler::resetUnregisterObservableForTest() { #endif void Profiler::unregisterThreadInEngines(int tid) { - HookCallScope scope(_instance->_hook_calls_inflight); - // Gated only while start() may replace the per-thread tables, not on - // _engines_ready: a thread started during a recording can exit after stop(), - // and PerfEvents keeps its fd open until this call closes it. + HookCallScope scope(_instance->_unregister_calls_inflight); + // Closed only inside a ThreadTablesFreeze (see _thread_tables_stable). if (!_instance->_thread_tables_stable.load(std::memory_order_seq_cst)) { return; } @@ -323,11 +325,8 @@ void Profiler::unregisterThreadInEngines(int tid) { g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); return; #endif - // start() may replace the engines concurrently. Either engine - // is safe to call: unregisterThread() only releases the exiting thread's own - // slot, bound-checked against the engine's current table, and a CTimer table - // being replaced is behind the gate above. PerfEvents replaces its table - // without the gate (PROF-16359). + // start() may replace the engine slots concurrently. Either engine is safe + // to call: unregisterThread() releases only the exiting thread's own slot. __atomic_load_n(&_instance->_cpu_engine, __ATOMIC_ACQUIRE) ->unregisterThread(tid); __atomic_load_n(&_instance->_wall_engine, __ATOMIC_ACQUIRE) @@ -1146,30 +1145,67 @@ void Profiler::disableEngines() { _wall_engine->enableEvents(false); } -// The registration gate is open for a recording only if the CPU engine -// started. An engine that failed or was not started must never see the hook: -// its registerThread() would arm a timer or open a perf fd for it. Engines with -// a per-thread table (CTimer, PerfEvents) open the gate themselves before they -// list threads, so opening it again here does nothing for them. For engines -// without a table (ITimer, ITimerJvmti, J9) it opens the gate, so registration -// follows the profiler lifecycle rather than the engine choice. The gate admits -// only the CPU engine (registerThreadInEngines()): no wall engine overrides -// Engine::registerThread(), so a recording whose CPU engine did not start -// loses nothing with the gate closed. +// Keeps the registration gate open iff the CPU engine started (see +// _engines_ready): an engine that failed must never see the hook. Opening it +// here is a no-op for engines that opened it in their start() (CTimer, +// PerfEvents), and opens it for the table-less ones (ITimer, ITimerJvmti, J9). bool Profiler::startCpuEngine(Arguments &args) { Error error = _cpu_engine->start(args); if (error) { - // The engine may have opened the gate before failing. A call that is - // still in flight after the wait budget is waited for again by the - // ThreadTablesFreeze in front of the next table replacement. - (void)closeThreadRegistration(); Log::warn("%s", error.message()); + // The engine may have opened the gate before failing. If a registration + // call is still inside it after the wait budget, that call may keep using + // the engine's table, and only start() replaces tables. ERROR rejects + // every later start(); the caller undoes what its start() set up. A + // failed table freeze in the engine's start() ends here too when the + // stuck call is a registration call; otherwise the failure is retryable. + if (!closeThreadRegistration()) { + Log::error("Profiler: a thread registration call did not leave the CPU " + "engine; the profiler is disabled"); + _state.store(ERROR, std::memory_order_release); + } return false; } openThreadRegistration(); return true; } +// Undoes what start() set up once _jfr.start() succeeded, when start() then +// fails without a running recording: engine events, the library trap and the +// refresher, the JFR recording, and the library patches. The patches are +// restored after JFR stops, as in stop(); a later start() patches again from +// Libraries::updateSymbols(). Without this, a start() that left the profiler +// in ERROR would leave the hooks patched for the rest of the process. +void Profiler::rollBackStart() { + disableEngines(); + switchLibraryTrap(false); + _libs->stopRefresher(); + lockAll(); + _jfr.stop(); + unlockAll(); + LibraryPatcher::unpatch_libraries(); +} + +// stop() leaves an engine's per-thread resources in place (PerfEvents keeps +// its fds so that unregisterThread() closes them when threads exit). Once the +// slot points at another engine, thread exits no longer reach the previous +// one, so it releases them first. The freeze keeps registerThread() and +// unregisterThread() off both engines meanwhile. +bool Profiler::replaceCpuEngine(Engine *next) { + Engine *previous = _cpu_engine; + if (previous != NULL && previous != next) { + ThreadTablesFreeze freeze(this); + if (!freeze.drained()) { + return false; + } + previous->releaseThreads(); + __atomic_store_n(&_cpu_engine, next, __ATOMIC_RELEASE); + return true; + } + __atomic_store_n(&_cpu_engine, next, __ATOMIC_RELEASE); + return true; +} + void Profiler::enableCpuEngine(int activated) { // Only when the CPU engine itself started: another mode may have activated // after it failed, and fds or timers left armed by an earlier recording @@ -1778,14 +1814,16 @@ Error Profiler::start(Arguments &args, bool reset) { _thread_filter.remove(slot_id); // Remove from filtering initially (matches onThreadStart behavior) } - // Registration stays closed until the CPU engine started (see - // startCpuEngine()), even if an earlier stop() left it open. Unregistration - // stays open: a thread exit dropped while the profiler is alive leaves a - // stale slot, so only the table replacement inside the CPU engine's start() - // closes it (ThreadTablesFreeze). No wait here: nothing below touches a - // per-thread table before that freeze waits for the in-flight calls. + // Registration stays closed until the CPU engine starts; unregistration + // stays open (see _engines_ready). No wait here: only replaceCpuEngine() and + // the CPU engine's table sizing touch per-thread tables, each behind its own + // ThreadTablesFreeze. _engines_ready.store(false, std::memory_order_seq_cst); - __atomic_store_n(&_cpu_engine, selectCpuEngine(args), __ATOMIC_RELEASE); + if (!replaceCpuEngine(selectCpuEngine(args))) { + // Nothing replaced or released yet. Retryable, like the earlier error + // returns in this function. + return Error("thread hook calls did not drain; retry start()"); + } __atomic_store_n(&_wall_engine, selectWallEngine(args), __ATOMIC_RELEASE); _cstack = args._cstack; _force_jmethodID = args._force_jmethodID; @@ -1865,6 +1903,12 @@ Error Profiler::start(Arguments &args, bool reset) { startCpuEngine(args)) { activated |= EM_CPU; } + if (state() == ERROR) { + // startCpuEngine() could not close the registration gate after the CPU + // engine failed. Nothing else has started; undo the setup above. + rollBackStart(); + return Error("thread hook calls did not drain; the profiler is disabled"); + } if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { // Enable wall clock BEFORE starting its pthread: timerLoopCommon() checks // _enabled once at startup and exits early if false, so the pthread would @@ -1915,12 +1959,7 @@ Error Profiler::start(Arguments &args, bool reset) { Log::warn("%s", error.message()); if (_event_mask == EM_NATIVEMEM) { // nativemem is the only requested mode: propagate the real error - disableEngines(); - switchLibraryTrap(false); - _libs->stopRefresher(); - lockAll(); - _jfr.stop(); - unlockAll(); + rollBackStart(); return error; } error = Error::OK; // recoverable when other modes are also active @@ -1975,13 +2014,7 @@ Error Profiler::start(Arguments &args, bool reset) { return Error::OK; } // no engine was activated; perform cleanup - disableEngines(); - switchLibraryTrap(false); - _libs->stopRefresher(); - - lockAll(); - _jfr.stop(); - unlockAll(); + rollBackStart(); return Error( "Neither CPU, wallclock nor allocation profiling could be started"); @@ -2009,23 +2042,20 @@ Error Profiler::stop() { return Error("signal handlers did not drain; teardown skipped, retry stop()"); } - // Close the registration gate and wait for calls already past it, so no - // thread arms a timer or opens an fd after the engines below stop, and - // none is still inside a table when the next start() resizes it. JVMTI - // ThreadStart goes through the same gate. Only registration is closed: - // JVMTI ThreadEnd and native thread exits may still reach unregisterThread() - // while the engines stop. That is safe per engine: CTimer::stop() and - // CTimer::unregisterThread() each release a slot with a per-slot CAS and - // neither frees the table; PerfEvents::stop() is a no-op, so it touches no - // slot an unregistration releases; wall engines and ITimer have no - // unregisterThread(). Only a CPU engine's start() replaces its table: - // CTimer under ThreadTablesFreeze, PerfEvents without it (PROF-16359). - // Unregistration stays open so PerfEvents can close the fd of a thread that - // exits after stop(). - if (!closeThreadRegistration()) { - // Same retry contract as the SignalInflight timeout above: state stays - // RUNNING, no engine stop has run, and a retry re-runs every step. - return Error("thread hook calls did not drain; teardown skipped, retry stop()"); + // Close the registration gate and wait for the calls already past it, so no + // thread arms a timer or opens an fd after the engines below stop. + // Unregistration stays open (see _thread_tables_stable). The engine stops + // below tolerate a concurrent unregisterThread() because none of them frees + // a per-thread table: CTimer releases slots with a per-slot CAS, and + // PerfEvents::stop() is a no-op. For the same reason the stop still + // completes if a registration call is stuck inside an engine after the wait + // budget; the profiler then ends in ERROR, which rejects every later + // start(), the only place a table is replaced. Unregistration calls are not + // waited for here: they may keep arriving during stop() under thread churn. + bool hooks_drained = closeThreadRegistration(); + if (!hooks_drained) { + Log::error("Profiler: thread hook calls did not drain; stopping and " + "disabling the profiler"); } if (_event_mask & EM_ALLOC) @@ -2148,6 +2178,10 @@ Error Profiler::stop() { // owned by library metadata, so we must keep library patches active until after serialization LibraryPatcher::unpatch_libraries(); + if (!hooks_drained) { + _state.store(ERROR, std::memory_order_release); + return Error("thread hook calls did not drain; the profiler is stopped and disabled"); + } _state.store(IDLE, std::memory_order_release); return Error::OK; } diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 6aed0cde0f..d42b81dc4b 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -89,37 +89,52 @@ class alignas(alignof(SpinLock)) Profiler { Mutex _state_lock; std::atomic _state; - // Registration gate, set by openThreadRegistration() and cleared by - // start(), closeThreadRegistration() and ThreadTablesFreeze. registerThread() is - // reached from the pthread hook without _state_lock, so it checks this flag - // to stay off the engines while start() sizes their per-thread structures (for example - // CTimer::resizeTimers()) and once stop() has torn them down. + // Thread-registration gates. The pthread_create hook and the JVMTI + // ThreadStart/ThreadEnd callbacks reach the engines only through + // registerThread()/unregisterThread(), without _state_lock. The two gates + // keep them off the per-thread tables that start() sizes, replaces or + // releases. This is the reference for the contract in code; the rest is in + // doc/architecture/ProfilingEngines.md#thread-registration. + // + // Registration gate: open iff the CPU engine started, and it admits only + // that engine; a wall engine that needs registerThread() needs its own gate. + // Set by openThreadRegistration(), from the CPU engine's start() once its + // table is sized and from startCpuEngine() on success. Cleared by start(), + // closeThreadRegistration() (stop(), failed CPU engine start) and + // ThreadTablesFreeze. std::atomic _engines_ready; - // Unregistration gate. Cleared by ThreadTablesFreeze only while a CPU - // engine replaces its per-thread table, and set again once the table is - // sized. Unlike _engines_ready it stays set while the profiler is idle: - // PerfEvents keeps its per-thread fds across stop(), and only - // unregisterThread() closes the fd of a thread that exits after stop(). + // Unregistration gate: open except inside a ThreadTablesFreeze. It stays + // open while idle, because PerfEvents keeps its per-thread fds across + // stop() and only unregisterThread() closes the fd of a thread that exits + // after stop(). An exit that arrives while it is closed is dropped, and its + // slot may stay occupied (PROF-16359). std::atomic _thread_tables_stable; - // Hook calls that are past either gate and may still be inside an engine. - // Closing a gate waits for this to reach zero. Kept off _state's cache line: - // the hook writes it on every native thread start and exit, while - // isRunning() reads _state on every intercepted socket call. - alignas(DEFAULT_CACHE_LINE_SIZE) std::atomic _hook_calls_inflight; + // Hook calls past a gate that may still be inside an engine, counted per + // gate. closeThreadRegistration() waits only for registration calls: + // unregistration stays open during stop(), and under thread churn its + // count may not reach zero within the wait budget, although no call is + // stuck. ThreadTablesFreeze closes both gates and waits for both counts. + // Kept off _state's cache line: the hook writes them on every native thread + // start and exit, while isRunning() reads _state on every intercepted + // socket call. + alignas(DEFAULT_CACHE_LINE_SIZE) std::atomic _register_calls_inflight; + std::atomic _unregister_calls_inflight; // Wait budget of waitForHookCalls(); HOOK_CALLS_DRAIN_TIMEOUT_NS (1 s) in // profiler.cpp. Written only by the friend test accessor. long long: test // budgets of several seconds must not overflow a 32-bit long. static long long _hook_drain_timeout_ns; - // Closes the gate opened by openThreadRegistration() and waits for hook - // calls already past it to leave the engines. Returns false if calls are - // still in flight after the wait budget (logged); the caller must not tear - // down or replace any table then. + // Closes the registration gate and waits for the registration calls already + // past it to leave the engines. Returns false if one is still in flight + // after the wait budget (logged); the caller must not free or replace any + // table then. stop() and startCpuEngine() put the profiler in ERROR on that + // result, so no later start() replaces a table under the stuck call. bool closeThreadRegistration(); - // Waits until no hook call is inside an engine, for at most + // Waits until no registration call (and, with include_unregistration, no + // unregistration call) is inside an engine, for at most // _hook_drain_timeout_ns. Returns false on timeout (logged); the caller - // must not tear down or replace any table then. - bool waitForHookCalls(); + // must not free or replace any table then. + bool waitForHookCalls(bool include_unregistration); // Gate checks and engine calls behind registerThread()/unregisterThread(), // run with thread cancellation disabled. static int registerThreadInEngines(int tid); @@ -208,8 +223,16 @@ class alignas(alignof(SpinLock)) Profiler { void disableEngines(); // Starts _cpu_engine. On success opens thread registration (idempotent for // engines that already opened it); on failure closes it again and logs the - // error. Returns whether the engine started. + // error. If closing times out it also puts the profiler in ERROR. Returns + // whether the engine started. bool startCpuEngine(Arguments &args); + // Installs next as _cpu_engine. When it replaces another engine, the + // previous one releases the per-thread resources it keeps across stop() + // (Engine::releaseThreads()) under a ThreadTablesFreeze. Returns false, + // replacing nothing, if hook calls do not drain. + bool replaceCpuEngine(Engine *next); + // Undoes start()'s setup after _jfr.start() when start() then fails. + void rollBackStart(); // Enables the CPU engine's events iff EM_CPU is in activated. void enableCpuEngine(int activated); @@ -293,26 +316,21 @@ class alignas(alignof(SpinLock)) Profiler { } public: - // Keeps Profiler::registerThread() and unregisterThread() off the engines - // while a CPU engine replaces its per-thread table. CTimer constructs it in - // start() only when the table size changes (pid_max changed, or no table - // yet): a thread exit skipped while it is alive is dropped, so it must not - // span a restart that keeps the table. PerfEvents::start() still replaces its - // table without it (PROF-16359). Both gates are set again by - // openThreadRegistration() once the table is sized; the destructor reopens - // unregistration at the latest. An exit skipped in this window is cleaned up - // only if the thread is gone when the engine lists threads; a thread that is - // still listed while it exits is registered by start() and its slot stays - // occupied after it dies (PROF-16359). + // Closes both thread-registration gates (see _engines_ready) and waits for + // the hook calls already past them. Construct it only around an actual + // table replacement (CTimer::sizeTimerTable()) or a CPU engine change + // (replaceCpuEngine()), never for a whole start(): exits are dropped while + // it is alive. The destructor reopens unregistration; registration reopens + // only when the CPU engine starts. class ThreadTablesFreeze { public: explicit ThreadTablesFreeze(Profiler *profiler) : _profiler(profiler), _drained(false) { // Both gates: registerThread() must not reach a table being replaced - // either, even if an earlier stop() left _engines_ready set. + // or released either. _profiler->_engines_ready.store(false, std::memory_order_seq_cst); _profiler->_thread_tables_stable.store(false, std::memory_order_seq_cst); - _drained = _profiler->waitForHookCalls(); + _drained = _profiler->waitForHookCalls(/*include_unregistration=*/true); } ~ThreadTablesFreeze() { _profiler->_thread_tables_stable.store(true, std::memory_order_seq_cst); @@ -330,7 +348,8 @@ class alignas(alignof(SpinLock)) Profiler { Profiler() : _state_lock(), _state(State::NEW), _engines_ready(false), - _thread_tables_stable(true), _hook_calls_inflight(0), + _thread_tables_stable(true), _register_calls_inflight(0), + _unregister_calls_inflight(0), _class_unload_hook_trap(2), _notify_class_unloaded_func(NULL), _thread_info(), _class_map(1), _string_label_map(2), _context_value_map(3), _thread_filter(), @@ -457,22 +476,11 @@ class alignas(alignof(SpinLock)) Profiler { void logStats(); void switchThreadEvents(jvmtiEventMode mode); /** - * Lets the native-thread hook's registerThread() reach the CPU engine, and - * reopens unregistration. A CPU engine with a per-thread table (CTimer, PerfEvents) - * calls this from its start() once the table is sized and before it lists - * the existing threads, so the hook handles a thread that starts or exits - * while start() registers the existing ones. Profiler::startCpuEngine() - * calls it again once the CPU engine's start() succeeds, and closes the - * gate if it fails, so the gate is open iff the CPU engine started; - * wall-only, noop and failed-CPU recordings leave it closed. Because the - * gate tracks the CPU engine, registerThread() reaches only that engine. A - * wall engine that needs registerThread() needs its own gate; opening this - * one would also admit a CPU engine that failed. For CTimer - * this misses no thread: the per-slot CAS resolves the hook and start() - * registering one tid. PerfEvents::start()'s stale-fd sweep may close the - * fd of a tid the hook registered after the thread list was taken, and - * such a thread is then not sampled for the recording (PROF-16359). - * Engines without registerThread() never need the gate. Idempotent. + * Opens both thread-registration gates (see _engines_ready). A CPU engine + * with a per-thread table calls it from its start() once the table is sized + * and before it lists the existing threads, so the hook covers a thread + * that starts or exits while start() registers the existing ones. + * startCpuEngine() calls it again once the CPU engine started. Idempotent. */ void openThreadRegistration(); diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 011ea254ca..2011376ebd 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -50,8 +50,10 @@ class ProfilerTestAccessor { static bool enginesReady(Profiler *p) { return p->_engines_ready.load(std::memory_order_acquire); } + // Registration and unregistration calls in flight, together. static int hookCallsInflight(Profiler *p) { - return p->_hook_calls_inflight.load(std::memory_order_acquire); + return p->_register_calls_inflight.load(std::memory_order_acquire) + + p->_unregister_calls_inflight.load(std::memory_order_acquire); } static bool closeThreadRegistration(Profiler *p) { return p->closeThreadRegistration(); @@ -62,6 +64,25 @@ class ProfilerTestAccessor { static void enableCpuEngine(Profiler *p, int activated) { p->enableCpuEngine(activated); } + static bool replaceCpuEngine(Profiler *p, Engine *next) { + return p->replaceCpuEngine(next); + } + static Engine *cpuEngine(Profiler *p) { return p->_cpu_engine; } + static State state(Profiler *p) { return p->state(); } + + // Restores the profiler state on destruction, so a test that drives the + // singleton into ERROR does not leak that into later tests. + class StateRestore { + public: + explicit StateRestore(Profiler *p) : _p(p), _state(p->state()) {} + ~StateRestore() { _p->_state.store(_state, std::memory_order_release); } + StateRestore(const StateRestore &) = delete; + StateRestore &operator=(const StateRestore &) = delete; + + private: + Profiler *_p; + State _state; + }; // Runs body(drained) while a real ThreadTablesFreeze is alive. template static void withThreadTablesFreeze(Profiler *p, F body) { Profiler::ThreadTablesFreeze freeze(p); @@ -82,21 +103,30 @@ class ProfilerTestAccessor { private: long long _saved; }; - // Counts a hook call in flight for its lifetime, as if one were stuck inside - // an engine, without needing a thread or an engine to hold it. - class HookCallHold { + // Counts a hook call in flight on one gate's counter for its lifetime, as if + // one were stuck inside an engine, without a thread or an engine to hold it. + class InflightHold { public: - explicit HookCallHold(Profiler *p) : _p(p) { - p->_hook_calls_inflight.fetch_add(1, std::memory_order_seq_cst); - } - ~HookCallHold() { - _p->_hook_calls_inflight.fetch_sub(1, std::memory_order_release); + explicit InflightHold(std::atomic &counter) : _counter(counter) { + _counter.fetch_add(1, std::memory_order_seq_cst); } - HookCallHold(const HookCallHold &) = delete; - HookCallHold &operator=(const HookCallHold &) = delete; + ~InflightHold() { _counter.fetch_sub(1, std::memory_order_release); } + InflightHold(const InflightHold &) = delete; + InflightHold &operator=(const InflightHold &) = delete; private: - Profiler *_p; + std::atomic &_counter; + }; + // A registration call held in flight. + class HookCallHold : public InflightHold { + public: + explicit HookCallHold(Profiler *p) : InflightHold(p->_register_calls_inflight) {} + }; + // An unregistration call held in flight. + class UnregisterCallHold : public InflightHold { + public: + explicit UnregisterCallHold(Profiler *p) + : InflightHold(p->_unregister_calls_inflight) {} }; static bool threadTablesStable(Profiler *p) { return p->_thread_tables_stable.load(std::memory_order_acquire); @@ -316,13 +346,17 @@ TEST_F(CTimerNativeMemTest, FailedAllocationLeavesEmptyTable) { // so a thread that starts or exits while start() registers the existing // threads is handled by the pthread hook instead of being missed. The engine // is stopped, the gate closed and the SIGPROF disposition restored before the -// fixture restores the table, so later tests see the process as it was. +// fixture restores the table, so later tests see the process as it was. The +// engine is installed as the Profiler's CPU engine while the gate is open, so +// a hook call in that window reaches it instead of a null engine slot. TEST_F(CTimerNativeMemTest, StartOpensThreadRegistration) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); struct sigaction saved_sigprof; sigaction(SIGPROF, nullptr, &saved_sigprof); CTimer engine; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &engine, &wall); Arguments args; Error error = CTimerTestAccessor::startWithTableSize(engine, args, @@ -395,7 +429,7 @@ TEST_F(CTimerNativeMemTest, ResizeKeepsTableWhenHookCallsDoNotDrain) { } EXPECT_TRUE((bool)error); - EXPECT_STREQ("thread hook calls did not drain; retry start()", + EXPECT_STREQ("thread hook calls did not drain", error.message()); EXPECT_EQ(table, CTimerTestAccessor::timers()); EXPECT_EQ(64, CTimerTestAccessor::maxTimers()); @@ -557,6 +591,44 @@ TEST(ProfilerEnginesGateTest, CloseThreadRegistrationTimesOutOnStuckHookCall) { EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); } +// stop() keeps unregistration open, and under thread churn unregistration +// calls can overlap without pause. closeThreadRegistration() therefore waits +// only for registration calls: an unregistration in flight does not make +// stop() time out and put the profiler in ERROR. +TEST(ProfilerEnginesGateTest, CloseThreadRegistrationIgnoresInFlightUnregistration) { + Profiler *profiler = Profiler::instance(); + ProfilerTestAccessor::RegistrationGate gate(profiler); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + bool closed; + { + ProfilerTestAccessor::UnregisterCallHold hold(profiler); + closed = ProfilerTestAccessor::closeThreadRegistration(profiler); + } + + EXPECT_TRUE(closed); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// A table replacement or a CPU engine change must exclude unregistration as +// well, so ThreadTablesFreeze also waits for unregistration calls. +TEST(ProfilerEnginesGateTest, ThreadTablesFreezeWaitsForInFlightUnregistration) { + Profiler *profiler = Profiler::instance(); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + bool drained = true; + { + ProfilerTestAccessor::UnregisterCallHold hold(profiler); + ProfilerTestAccessor::withThreadTablesFreeze( + profiler, [&](bool d) { drained = d; }); + } + + EXPECT_FALSE(drained); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + // The ThreadTablesFreeze a CPU engine's start() takes around a table // replacement closes both gates before it waits for hook // calls already inside an engine, does not return while one is, keeps @@ -687,6 +759,7 @@ TEST(ProfilerEnginesGateTest, FailedCpuEngineStartClosesThreadRegistration) { Engine wall; ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); Arguments args; + State state_before = ProfilerTestAccessor::state(profiler); bool started = ProfilerTestAccessor::startCpuEngine(profiler, args); bool ready = ProfilerTestAccessor::enginesReady(profiler); @@ -701,6 +774,109 @@ TEST(ProfilerEnginesGateTest, FailedCpuEngineStartClosesThreadRegistration) { EXPECT_EQ(0, cpu.register_calls.load()); EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + // The gate closed within the budget, so the profiler stays usable. + EXPECT_EQ(state_before, ProfilerTestAccessor::state(profiler)); +} + +// If the gate cannot be closed after the CPU engine failed, a hook call may +// still be inside that engine's table, and only start() replaces tables. The +// profiler goes to ERROR, which rejects every later start(). +TEST(ProfilerEnginesGateTest, StuckGateAfterCpuEngineFailurePutsProfilerInError) { + Profiler *profiler = Profiler::instance(); + ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + ProfilerTestAccessor::StateRestore restore(profiler); + FakeCpuEngine cpu(/*fail=*/true, /*open_gate=*/true); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &cpu, &wall); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + Arguments args; + + bool started; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + started = ProfilerTestAccessor::startCpuEngine(profiler, args); + } + + EXPECT_FALSE(started); + EXPECT_EQ(ERROR, ProfilerTestAccessor::state(profiler)); + EXPECT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// A CPU engine that counts releaseThreads() calls and records the gates it +// saw while releasing. +class ReleasingEngine : public Engine { +public: + std::atomic release_calls{0}; + std::atomic stable_inside{true}; + std::atomic ready_inside{true}; + + void releaseThreads() override { + release_calls.fetch_add(1); + stable_inside.store( + ProfilerTestAccessor::threadTablesStable(Profiler::instance())); + ready_inside.store( + ProfilerTestAccessor::enginesReady(Profiler::instance())); + } +}; + +// stop() leaves an engine's per-thread resources in place (PerfEvents keeps +// its fds open), and only unregisterThread() releases them as threads exit. +// When start() selects another CPU engine, exits no longer reach the previous +// one, so it releases them before the slot changes, with both gates closed. +TEST(ProfilerEnginesGateTest, CpuEngineChangeReleasesPreviousEngineThreads) { + Profiler *profiler = Profiler::instance(); + ReleasingEngine previous; + FakeCpuEngine next(/*fail=*/false, /*open_gate=*/false); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &previous, &wall); + + bool replaced = ProfilerTestAccessor::replaceCpuEngine(profiler, &next); + + EXPECT_TRUE(replaced); + EXPECT_EQ(1, previous.release_calls.load()); + EXPECT_FALSE(previous.stable_inside.load()); + EXPECT_FALSE(previous.ready_inside.load()); + EXPECT_EQ(&next, ProfilerTestAccessor::cpuEngine(profiler)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); + EXPECT_EQ(0, ProfilerTestAccessor::hookCallsInflight(profiler)); +} + +// Restarting with the same CPU engine keeps its per-thread resources, so +// PerfEvents reuses its fds across stop()/start(). +TEST(ProfilerEnginesGateTest, SameCpuEngineKeepsItsThreads) { + Profiler *profiler = Profiler::instance(); + ReleasingEngine engine; + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &engine, &wall); + + bool replaced = ProfilerTestAccessor::replaceCpuEngine(profiler, &engine); + + EXPECT_TRUE(replaced); + EXPECT_EQ(0, engine.release_calls.load()); + EXPECT_EQ(&engine, ProfilerTestAccessor::cpuEngine(profiler)); +} + +// If hook calls do not drain, the engine change is refused: the previous +// engine keeps its resources and its slot, and start() fails retryably. +TEST(ProfilerEnginesGateTest, CpuEngineChangeKeepsPreviousEngineWhenHookCallsDoNotDrain) { + Profiler *profiler = Profiler::instance(); + ReleasingEngine previous; + FakeCpuEngine next(/*fail=*/false, /*open_gate=*/false); + Engine wall; + ProfilerTestAccessor::EngineSwap swap(profiler, &previous, &wall); + ProfilerTestAccessor::DrainBudget budget(100000000LL); + + bool replaced; + { + ProfilerTestAccessor::HookCallHold hold(profiler); + replaced = ProfilerTestAccessor::replaceCpuEngine(profiler, &next); + } + + EXPECT_FALSE(replaced); + EXPECT_EQ(0, previous.release_calls.load()); + EXPECT_EQ(&previous, ProfilerTestAccessor::cpuEngine(profiler)); + EXPECT_TRUE(ProfilerTestAccessor::threadTablesStable(profiler)); } // A CPU engine without a per-thread table (ITimer-like) never opens the gate diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index 59bb7c6915..cf702738df 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -53,4 +53,15 @@ TEST(PerfEventsCompareTidsTest, SortsCreationOrderedTidsWithReuse) { } } +// A failed table allocation leaves _events NULL and _max_events -1, which is +// also the state before the first start(); no test in this binary starts +// PerfEvents. Fds left armed by an earlier recording keep delivering SIGPROF, +// and the handler calls resetBuffer() on every exit, so it must not index the +// missing table. releaseThreads() must not touch it either. +TEST(PerfEventsTableTest, MissingTableIsNotIndexed) { + PerfEvents::resetBuffer(5); + PerfEvents engine; + engine.releaseThreads(); +} + #endif // __linux__ diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index 2a8eef3061..ee744682af 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -69,16 +69,21 @@ Profiler::start() _engines_ready cleared registration closed until the CPU engine starts; unregistration stays open selectCpuEngine / selectWallEngine engine pointers stored with release + replaceCpuEngine() on an engine change, the previous engine + releases its threads under + ThreadTablesFreeze; timeout -> retryable error _jfr.start() SamplerPerf::primeClock() before any handler can fire Profiler::startCpuEngine() (EM_CPU) failure -> warning, continue _cpu_engine->start() CTimer: ThreadTablesFreeze only if the table size changes; - on a wait timeout (1 s) it fails retryably and keeps - its table - PerfEvents: replaces its table without the freeze (PROF-16359) + on a wait timeout (1 s) it fails and keeps its + table (ERROR below if a registration call is stuck) + PerfEvents: replaces its table without the freeze (PROF-16359); + allocation failure -> error, engine stays off CTimer/PerfEvents: openThreadRegistration() after sizing, before listing threads - on failure: closeThreadRegistration() + on failure: closeThreadRegistration(); timeout -> ERROR, start() undoes + its setup and fails on success: openThreadRegistration() no-op for CTimer/PerfEvents _wall_engine->enableEvents(true) BEFORE start(): see below _wall_engine->start() (EM_WALL) @@ -107,13 +112,18 @@ Profiler::start() 2. `SignalInflight::drain()` waits until every handler that already passed its `_enabled` check has left. If the drain times out, `stop()` returns an error and tears nothing down. -3. `closeThreadRegistration()` clears `_engines_ready` and waits until no hook - call that passed the check is still inside an engine. +3. `closeThreadRegistration()` clears `_engines_ready` and waits until no + registration call that passed the check is still inside an engine. No `registerThread()` call (pthread hook or JVMTI `ThreadStart`) can then arm a timer or open an fd after the engines stop, or still be using a table when the next `start()` resizes it. The wait is bounded (1 s). On timeout `stop()` - returns an error, tears nothing down and keeps the state `RUNNING`, like - step 2; a retried `stop()` runs every step again. + logs an error, still completes every step below, and ends in `ERROR` instead + of `IDLE`. That is safe because no step frees a per-thread table, and + `ERROR` rejects every later `start()`, the only place a table is replaced, + so the stuck call can never reach freed memory. The cost: if the stuck call + is inside `CTimer::registerThread()` and finishes after `CTimer::stop()`, + its timer stays armed for good; its `SIGPROF` is dropped by the disabled + handler. Unregistration is not closed here: JVMTI `ThreadEnd` and native thread exits may still reach `unregisterThread()` while the engines stop. `CTimer::stop()` and `CTimer::unregisterThread()` each release a slot with a per-slot CAS and @@ -178,18 +188,22 @@ Per-thread engines (`CTimer`, `PerfEvents`) learn about threads in three ways: with acquire. An unregistration that reaches the previous engine only releases that thread's own slot. `check()` never writes the slots. 3. **`PerfEvents` only**: a `pthread_setspecific` GOT hook on the JVM library - (`perfEvents_linux.cpp:176`), installed in `PerfEvents::start()` - (`perfEvents_linux.cpp:980`). + (`pthread_setspecific_hook()`), installed in `PerfEvents::start()`. Every `Profiler::registerThread()`/`unregisterThread()` call counts itself in -flight (`HookCallScope`) for as long as it may be inside an engine. Closing a gate -(`closeThreadRegistration()` in `stop()`, `ThreadTablesFreeze` around a table -replacement) waits -for that count to reach zero, at most 1 s, so no `registerThread()` call is inside -an engine once `stop()` tears it down, and no call at all while `start()` replaces -its table. If the wait times out, the caller returns a retryable error and -neither tears down nor replaces anything. The gate flags and the counter use `seq_cst` on both -sides: either a call sees the gate closed, or the closer sees the call and waits. +flight (`HookCallScope`) for as long as it may be inside an engine, on its +gate's own counter (`_register_calls_inflight`, `_unregister_calls_inflight`). +`closeThreadRegistration()` waits, at most 1 s, only for registration calls, so +no `registerThread()` call is inside an engine once `stop()` tears it down. +Unregistration calls are not waited for there: they stay allowed during `stop()`, +and under thread churn their count may not reach zero within the budget although +none is stuck. `ThreadTablesFreeze` closes both gates and waits for both counts, +so no call at all is inside an engine while a table is replaced or a replaced +CPU engine releases its threads. On a freeze timeout the operation fails and +replaces nothing; on a `closeThreadRegistration()` timeout the profiler goes to +`ERROR` (see `stop()` above and `Profiler::startCpuEngine()`). The gate flags and +the counters use `seq_cst` on both sides: either a call sees the gate closed, or +the closer sees the call and waits. Thread cancellation is disabled for the length of each call, because the engine calls reach cancellation points (`close()`, `Log::warn()`). Acting on a @@ -311,7 +325,7 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Thread hook**: `start()` patches the JVM library's `pthread_setspecific` import so that HotSpot setting or clearing its thread TLS key registers or unregisters the thread (`perfEvents_linux.cpp:171-204`). -- **`stop()` is a no-op** (`perfEvents_linux.cpp:1051`): fds stay open across +- **`stop()` is a no-op**: fds stay open across stop/start, and `registerThread()` skips threads that already have an fd. After stop, the cleared `_enabled` flag keeps handlers from recording. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted @@ -321,6 +335,15 @@ resolved through `Arguments::cpuSamplerInterval()`. `pid_max` changed. It does not close the old table's fds or rings, and it does not exclude thread unregistration or the signal-path readers (`walkKernel()`, `resetBuffer()`) while it does so (PROF-16359). +- **Allocation failure**: if the table cannot be allocated, `start()` sets + `_max_events` back to -1, logs an error and fails before it installs the + signal handler, the hook or the registration gate, so the engine stays off + for the recording. `walkKernel()` and `resetBuffer()` skip the missing table: + fds left armed by an earlier recording keep delivering `SIGPROF`. +- **Engine change**: `releaseThreads()` closes every fd and unmaps every ring. + `Profiler::replaceCpuEngine()` calls it when `start()` selects another CPU + engine, because thread exits stop reaching PerfEvents once it leaves the + slot. - `CSTACK_LBR` and kernel stacks (`walkKernel`) are available only with this engine (`Profiler::start()`, `Profiler::getNativeTrace()`). From 73f025555367f79d953e3ffcb5650c29313189c5 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 20:27:30 -0400 Subject: [PATCH 25/26] PROF-16213: Correct check() preconditions; add check() JVM test check() runs only while the profiler is IDLE: checkState() rejects every state above IDLE, so a RUNNING profiler is refused before any engine is selected. The comments said otherwise. The locals change matters after stop(): unregistration stays open then, and selecting into the engine slots would route the previous recording's exits to a different engine, leaking its PerfEvents fds. The comments and ProfilingEngines.md now say this. Fix the stale "Table replacement" note: start() releases the old fds and rings before it frees the table. Narrow MissingTableIsNotIndexed to what it checks; releaseThreads() on an empty table tests nothing. Add CheckCommandTest: check() is rejected while recording and the recording continues; check() succeeds after stop(). The engine slot itself is not observable from Java, so the test does not cover it. Co-Authored-By: Claude Haiku 5.5 --- ddprof-lib/src/main/cpp/profiler.cpp | 9 +++-- ddprof-lib/src/main/cpp/profiler.h | 3 +- ddprof-lib/src/test/cpp/perfEvents_ut.cpp | 12 +++--- .../datadoghq/profiler/CheckCommandTest.java | 40 +++++++++++++++++++ doc/architecture/ProfilingEngines.md | 15 +++---- 5 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 ddprof-test/src/test/java/com/datadoghq/profiler/CheckCommandTest.java diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 3da5b59f3e..87994d3e54 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -2195,10 +2195,11 @@ Error Profiler::check(Arguments &args) { error = checkJvmCapabilities(); - // check() may run while a recording is RUNNING (checkState() rejects only - // NEW and ERROR), so it selects into locals and leaves the engine slots to - // the engines that start() started: the hooks, the signal handlers (the wall - // handler casts wallEngine() to its own type) and stop() all use those. + // check() runs while the profiler is IDLE, after stop(): unregistration is + // still open then, and the previous recording's engine may hold fds that + // only its unregisterThread() closes. Selecting into the slots here would + // route those exits to the new engine and leak the fds, so check() selects + // into locals. checkState() rejects a RUNNING profiler before this point. Engine *cpu_engine = NULL; if (!error && (args._event != NULL || args._cpu >= 0)) { cpu_engine = selectCpuEngine(args); diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index d42b81dc4b..bcc90e27d5 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -168,7 +168,8 @@ class alignas(alignof(SpinLock)) Profiler { // Replaced only by start(), while thread unregistration is open, so the // hook paths (registerThreadInEngines(), unregisterThreadInEngines()) load // them with acquire and start() stores them with release. check() selects - // into locals: it may run during a recording. + // into locals and runs only while IDLE, where the slots still route exits + // to the previous recording's engine. Engine *_cpu_engine; Engine *_wall_engine = NULL; Engine *_alloc_engine; diff --git a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp index cf702738df..477584ed24 100644 --- a/ddprof-lib/src/test/cpp/perfEvents_ut.cpp +++ b/ddprof-lib/src/test/cpp/perfEvents_ut.cpp @@ -54,14 +54,14 @@ TEST(PerfEventsCompareTidsTest, SortsCreationOrderedTidsWithReuse) { } // A failed table allocation leaves _events NULL and _max_events -1, which is -// also the state before the first start(); no test in this binary starts -// PerfEvents. Fds left armed by an earlier recording keep delivering SIGPROF, -// and the handler calls resetBuffer() on every exit, so it must not index the -// missing table. releaseThreads() must not touch it either. +// also the state before the first start(). Fds left armed by an earlier +// recording keep delivering SIGPROF, and the handler calls resetBuffer() on +// every exit, so resetBuffer() must not index the missing table. This checks +// only that no crash happens. Checking that releaseThreads() closes real fds +// needs slot setters in perfEvents_linux.cpp, which this binary does not have +// (PROF-16359). TEST(PerfEventsTableTest, MissingTableIsNotIndexed) { PerfEvents::resetBuffer(5); - PerfEvents engine; - engine.releaseThreads(); } #endif // __linux__ diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/CheckCommandTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/CheckCommandTest.java new file mode 100644 index 0000000000..451bbde46d --- /dev/null +++ b/ddprof-test/src/test/java/com/datadoghq/profiler/CheckCommandTest.java @@ -0,0 +1,40 @@ +/* + * Copyright 2026, Datadog, Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.datadoghq.profiler; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * The check command validates a profiler command without starting anything. + * While a recording runs it is rejected, and the recording must carry on. After + * stop() it validates the command and leaves the profiler idle. + */ +public class CheckCommandTest extends AbstractProfilerTest { + + @Override + protected String getProfilerCommand() { + return "cpu=1ms"; + } + + @Test + public void checkIsRejectedWhileRecording() throws Exception { + IllegalStateException e = assertThrows(IllegalStateException.class, + () -> profiler.execute("check,cpu=1ms")); + assertEquals("Profiler already started", e.getMessage()); + assertTrue(profiler.execute("status").contains("Profiling is running")); + } + + @Test + public void checkSucceedsWhenIdle() throws Exception { + stopProfiler(); + assertTrue(profiler.execute("check,cpu=1ms").startsWith("OK")); + assertTrue(profiler.execute("status").contains("Profiler is not active")); + } +} diff --git a/doc/architecture/ProfilingEngines.md b/doc/architecture/ProfilingEngines.md index ee744682af..ea0c3fa5cb 100644 --- a/doc/architecture/ProfilingEngines.md +++ b/doc/architecture/ProfilingEngines.md @@ -136,9 +136,9 @@ Profiler::start() `_cpu_engine->stop()`. `Profiler::check()` runs the same selection and calls each engine's `check()` -without starting anything. It selects into locals and never replaces the -engine slots: it may run while a recording is `RUNNING`, and the hooks, the -signal handlers and `stop()` use the slots of the engines `start()` started. +without starting anything. It runs only while the profiler is `IDLE` +(`checkState()` rejects a `RUNNING` profiler) and selects into locals, so it +never replaces the engine slots that thread exits still route to. ### Thread registration @@ -331,10 +331,11 @@ resolved through `Arguments::cpuSamplerInterval()`. - **Memory**: the `_events` array and any 2-page mmap ring pages are accounted under `NM_PERF`. The array is counted at its full allocated size, which is mostly untouched address space (see Profiler Self-Accounting). -- **Table replacement**: `start()` frees and reallocates `_events` when - `pid_max` changed. It does not close the old table's fds or rings, and it - does not exclude thread unregistration or the signal-path readers - (`walkKernel()`, `resetBuffer()`) while it does so (PROF-16359). +- **Table replacement**: when `pid_max` changed, `start()` releases the old + table's fds and rings (`releaseThreads()`) and then frees and reallocates + `_events`. It does not exclude thread unregistration or the signal-path + readers (`walkKernel()`, `resetBuffer()`) while it does so: a handler that + already holds a slot pointer can still use the freed table (PROF-16359). - **Allocation failure**: if the table cannot be allocated, `start()` sets `_max_events` back to -1, logs an error and fails before it installs the signal handler, the hook or the registration gate, so the engine stays off From e389e30c1e83d73f655d28c3428b60317f224fce Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Fri, 30 Oct 2026 20:47:17 -0400 Subject: [PATCH 26/26] PROF-16213: Count every registration call; drop the registration pre-check registerThreadInEngines() read the registration gate with an acquire load before raising its in-flight counter, so a call could skip both the count and the seq_cst re-check. The acquire read has no happens-before edge from the thread that opened the gate unless the creator synchronized with start(). Raise the counter on every registration call and read the gate seq_cst, as unregistration already does. This costs one shared-counter RMW per thread creation. Rename RegisterThreadFastPathDoesNotSkipOpenGate, since there is no fast path left. Co-Authored-By: Claude Haiku 5.5 --- ddprof-lib/src/main/cpp/profiler.cpp | 16 ++++++---------- ddprof-lib/src/test/cpp/ctimer_ut.cpp | 8 ++++---- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 87994d3e54..b61a127f47 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -179,9 +179,8 @@ void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { // increment and the gate // load in the caller are seq_cst, pairing with the seq_cst gate store and // counter load on the closing side: either the hook sees the gate closed, or -// the closer sees the call in flight and waits for it. registerThreadInEngines() -// skips the scope when an acquire pre-check already sees the registration gate -// closed; the decision that matters is still the seq_cst pair. +// the closer sees the call in flight and waits for it. Registration and +// unregistration both raise the counter on every call. class HookCallScope { public: explicit HookCallScope(std::atomic &inflight) : _inflight(inflight) { @@ -283,13 +282,10 @@ void Profiler::unregisterThread(int tid) { } int Profiler::registerThreadInEngines(int tid) { - // Advisory pre-check: skips the shared counter RMW while idle. A stale - // "closed" read equals a call made just before the gate opened, which the - // CPU engine's start() covers by listing threads after opening it; a stale - // "open" read falls through to the seq_cst re-check below. - if (!_instance->_engines_ready.load(std::memory_order_acquire)) { - return -1; // not registered, as Engine::registerThread() reports it - } + // The counter is raised before the gate is read, both seq_cst, so the + // closing side either sees this call in flight or this call sees the gate + // closed. A thread created before the gate opens is covered by the CPU + // engine's start(), which lists threads after opening it. HookCallScope scope(_instance->_register_calls_inflight); if (!_instance->_engines_ready.load(std::memory_order_seq_cst)) { return -1; // not registered, as Engine::registerThread() reports it diff --git a/ddprof-lib/src/test/cpp/ctimer_ut.cpp b/ddprof-lib/src/test/cpp/ctimer_ut.cpp index 2011376ebd..a112720893 100644 --- a/ddprof-lib/src/test/cpp/ctimer_ut.cpp +++ b/ddprof-lib/src/test/cpp/ctimer_ut.cpp @@ -923,10 +923,10 @@ TEST(ProfilerEnginesGateTest, CpuEngineEnabledOnlyIfItStarted) { EXPECT_EQ(1, cpu.enable_true_calls.load()); } -// The idle fast path in registerThread() only short-cuts a closed gate: with -// the gate open the call still reaches the engine and returns its result, and -// the in-flight count is back at zero afterwards either way. -TEST(ProfilerEnginesGateTest, RegisterThreadFastPathDoesNotSkipOpenGate) { +// With the gate closed registerThread() returns before reaching the engine; +// with it open the call reaches the engine and returns its result. The +// in-flight count is back at zero afterwards either way. +TEST(ProfilerEnginesGateTest, RegisterThreadReachesEngineOnlyWhenGateOpen) { Profiler *profiler = Profiler::instance(); ASSERT_FALSE(ProfilerTestAccessor::enginesReady(profiler)); FakeCpuEngine cpu(/*fail=*/false, /*open_gate=*/false);