Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 82 additions & 54 deletions ddprof-lib/src/main/cpp/callTraceHashTable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,53 @@ CallTrace *CallTraceHashTable::findCallTrace(LongHashTable *table, u64 hash) {
return nullptr;
}

// Upper 32 bits = instance_id, lower 32 bits = slot_base + local slot (slot_base
// keeps the low bits unique across all LongHashTable generations of one tenure).
u64 CallTraceHashTable::traceIdForSlot(LongHashTable *table, u32 slot) {
// ACQUIRE pairs with the RELEASE store in setInstanceId() (aarch64, POWER).
u64 instance_id = _instance_id.load(std::memory_order_acquire);
return (instance_id << 32) | (table->slotBase() + slot);
}

// Gives up a slot claimed by put(). The trace is reset before the key is cleared:
// a prober that still sees the key never finds a stale trace, and the next
// claimer's markPreparing() CAS from nullptr succeeds.
static void abandonClaim(LongHashTable *table, u32 slot) {
Comment thread
kaahos marked this conversation as resolved.
Resolved
table->values()[slot].setTrace(nullptr);
__atomic_thread_fence(__ATOMIC_SEQ_CST);
__atomic_store_n(&table->keys()[slot], 0, __ATOMIC_RELEASE);
}

#ifdef UNIT_TEST
u32 CallTraceHashTable::firstSlotForTest(u64 hash) {
return HashProbe(hash, _table->capacity()).slot();
}

bool CallTraceHashTable::claimSlotForTest(u32 slot, u64 key) {
u64 expected = 0;
return __atomic_compare_exchange_n(&_table->keys()[slot], &expected, key, false,
__ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
}

bool CallTraceHashTable::markPreparingForTest(u32 slot) {
return _table->values()[slot].markPreparing();
}

u64 CallTraceHashTable::publishSlotForTest(u32 slot, int num_frames, ASGCT_CallFrame *frames,
bool truncated) {
CallTrace *trace = storeCallTrace(num_frames, frames, truncated, traceIdForSlot(_table, slot));
if (trace == nullptr) {
return CallTraceStorage::DROPPED_TRACE_ID;
}
_table->values()[slot].setTrace(trace);
return trace->trace_id;
}

void CallTraceHashTable::abandonSlotForTest(u32 slot) {
abandonClaim(_table, slot);
}
#endif

void CallTraceHashTable::expandTableIfNeeded(LongHashTable* table, u32 size) {
u32 capacity = table->capacity();

Expand Down Expand Up @@ -362,6 +409,12 @@ void CallTraceHashTable::expandTableIfNeeded(LongHashTable* table, u32 size) {
u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
bool truncated, u64 weight) {
u64 hash = calcHash(num_frames, frames, truncated);
if (hash == 0) {
// 0 marks an empty key (a zero-frame stack hashes to it), so such a stack
// would match every empty slot and wait on it forever.
Counters::increment(CALLTRACE_STORAGE_DROPPED);
return CallTraceStorage::DROPPED_TRACE_ID;
}

// ACQUIRE pairs with the ACQ_REL CAS in the expansion path below, ensuring
// that if another thread published a new (expanded) table we see its fully
Expand All @@ -379,53 +432,38 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
u32 slot = probe.slot();
while (true) {
u64 key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED);
if (key_value == hash) {
// Hash matches - wait for the preparing thread to complete
CallTrace* current_trace = table->values()[slot].acquireTrace();

// If another thread is preparing this slot, wait for completion
if (current_trace == CallTraceSample::PREPARING) {
// Wait for the preparing thread to complete, with timeout
int wait_cycles = 0;
const int MAX_WAIT_CYCLES = 1000; // ~1000 cycles should be enough for allocation

do {
// Brief spin-wait to allow preparing thread to complete
for (volatile int i = 0; i < 10; i++) {
spinPause(); // Architecture-specific pause instruction
}

current_trace = table->values()[slot].acquireTrace();
wait_cycles++;

// Check if key was cleared (preparation failed)
if (__atomic_load_n(&keys[slot], __ATOMIC_RELAXED) != hash) {
break; // Key cleared, restart search
}

} while (current_trace == CallTraceSample::PREPARING && wait_cycles < MAX_WAIT_CYCLES);

// If still preparing after timeout, something is wrong - continue search
if (current_trace == CallTraceSample::PREPARING) {
continue;
if (key_value == hash) {
// A nullptr or PREPARING trace means the key-CAS winner has not published
// yet. Every abandon path clears the key, so wait for a publish or a key change.
CallTrace *current_trace = table->values()[slot].acquireTrace();
for (int wait_cycles = 0;
(current_trace == nullptr || current_trace == CallTraceSample::PREPARING) &&
Comment thread
kaahos marked this conversation as resolved.
Resolved
wait_cycles < MAX_WAIT_CYCLES;
wait_cycles++) {
#ifdef UNIT_TEST
if (_wait_hook_for_test != nullptr) {
_wait_hook_for_test(_wait_hook_arg_for_test);
}
#endif
for (volatile int i = 0; i < 10; i++) {
spinPause();
}
current_trace = table->values()[slot].acquireTrace();
// Relaxed: this only ends the spin early; the re-check below decides.
if (__atomic_load_n(&keys[slot], __ATOMIC_RELAXED) != hash) {
break;
}
}

// Check final state after waiting
if (current_trace != nullptr && current_trace != CallTraceSample::PREPARING) {
// Trace is ready, use it
// The slot may have been abandoned and reclaimed by another stack. The
// ACQUIRE trace load orders this re-check after that stack's key CAS.
if (current_trace != nullptr && current_trace != CallTraceSample::PREPARING &&
__atomic_load_n(&keys[slot], __ATOMIC_ACQUIRE) == hash) {
return current_trace->trace_id;
} else {
// Trace is nullptr but hash exists - preparation failed
u64 recheck_key = __atomic_load_n(&keys[slot], __ATOMIC_ACQUIRE);
if (recheck_key != hash) {
continue; // Key was cleared, retry
}
Counters::increment(CALLTRACE_STORAGE_DROPPED);
return CallTraceStorage::DROPPED_TRACE_ID;
}
// The key changed, or the owner is still filling the slot: probe it again.
continue;
}

if (key_value == 0) {
u64 expected = 0;
if (!__atomic_compare_exchange_n(&keys[slot], &expected, hash, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) {
Expand Down Expand Up @@ -454,19 +492,9 @@ u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames,
}

if (trace == nullptr) {
// Generate unique trace ID: upper 32 bits = instance_id, lower 32 bits =
// slot_base + local slot (slot_base makes the low bits unique across
// all LongHashTable generations of one active tenure)
// ACQUIRE ordering synchronizes with RELEASE store in setInstanceId() to ensure
// visibility of new instance_id on weakly-ordered architectures (aarch64, POWER)
u64 instance_id = _instance_id.load(std::memory_order_acquire);
u64 trace_id = (instance_id << 32) | (table->slotBase() + slot);
trace = storeCallTrace(num_frames, frames, truncated, trace_id);
trace = storeCallTrace(num_frames, frames, truncated, traceIdForSlot(table, slot));
if (trace == nullptr) {
// Allocation failure - reset trace first, then clear key
table->values()[slot].setTrace(nullptr);
__atomic_thread_fence(__ATOMIC_SEQ_CST);
__atomic_store_n(&keys[slot], 0, __ATOMIC_RELEASE);
abandonClaim(table, slot);
Counters::increment(CALLTRACE_STORAGE_DROPPED);
return CallTraceStorage::DROPPED_TRACE_ID;
}
Expand Down
17 changes: 17 additions & 0 deletions ddprof-lib/src/main/cpp/callTraceHashTable.h
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,22 @@ class CallTraceHashTable {
static u64 nextGenerationCapacity(u32 capacity);
static bool wouldExceedSlotIdRange(u64 slot_base, u32 capacity);

// Spin iterations put() waits on an unpublished slot before re-probing it.
static constexpr int MAX_WAIT_CYCLES = 1000;

#ifdef UNIT_TEST
// Reached via CallTraceHashTableTestAccessor: put() calls the hook once per
// spin iteration, and the slot operations reproduce a concurrent owner's
// states on the current table.
void (*_wait_hook_for_test)(void *arg) = nullptr;
void *_wait_hook_arg_for_test = nullptr;
u32 firstSlotForTest(u64 hash);
bool claimSlotForTest(u32 slot, u64 key);
bool markPreparingForTest(u32 slot);
u64 publishSlotForTest(u32 slot, int num_frames, ASGCT_CallFrame *frames, bool truncated);
void abandonSlotForTest(u32 slot);
#endif

std::atomic<u64> _instance_id; // 64-bit instance ID for this hash table - atomic for thread-safe access
CallTraceStorage* _parent_storage; // Parent storage for RefCountGuard access

Expand All @@ -120,6 +136,7 @@ class CallTraceHashTable {
CallTrace *storeCallTrace(int num_frames, ASGCT_CallFrame *frames,
bool truncated, u64 trace_id);
CallTrace *findCallTrace(LongHashTable *table, u64 hash);
u64 traceIdForSlot(LongHashTable *table, u32 slot);
void decrementCounters();

void expandTableIfNeeded(LongHashTable* table, u32 size);
Expand Down
19 changes: 17 additions & 2 deletions ddprof-lib/src/main/cpp/nativeSocketSampler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ static thread_local PoissonSampler _recv_sampler;

#ifdef UNIT_TEST
void (*NativeSocketSampler::_record_observer_for_test)(int, u64, u64, ssize_t, u8) = nullptr;
void (*NativeSocketSampler::_record_sample_for_test)(ProfiledThread*) = nullptr;
#endif

// Marks the hook wrapper's own symbol as MARK_JAVA_PROFILER so native call-stack
Expand Down Expand Up @@ -281,13 +282,27 @@ void NativeSocketSampler::recordEvent(int fd, u64 t0, u64 t1, ssize_t bytes, u8
if (current == nullptr) {
Counters::increment(SAMPLES_DROPPED_THREAD_LOCAL);
} else {
Profiler::instance()->recordSample(NULL, (u64)bytes, current->tid(),
BCI_NATIVE_SOCKET, 0, &event);
recordProfilerSample(current, (u64)bytes, &event);
}

_rate_limiter.recordFire();
}

void NativeSocketSampler::recordProfilerSample(ProfiledThread* current, u64 bytes,
NativeSocketEvent* event) {
CriticalSection cs(current);
if (!cs.entered()) {
Comment thread
kaahos marked this conversation as resolved.
Resolved
return;
}
#ifdef UNIT_TEST
if (_record_sample_for_test != nullptr) {
_record_sample_for_test(current);
return;
}
#endif
Profiler::instance()->recordSample(NULL, bytes, current->tid(), BCI_NATIVE_SOCKET, 0, event);
}

ssize_t NativeSocketSampler::send_hook(int fd, const void* buf, size_t len, int flags) {
// Defensive guard against direct invocation outside of PLT dispatch (e.g. tests
// that obtain the static symbol address before LibraryPatcher::patch_socket_functions
Expand Down
9 changes: 9 additions & 0 deletions ddprof-lib/src/main/cpp/nativeSocketSampler.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ class LibraryPatcher;
// risk because hook bodies do NOT acquire the profiler
// signal lock.

class ProfiledThread;

class NativeSocketSampler : public Engine {
public:
// Typedefs for libc send/recv/write/read signatures.
Expand Down Expand Up @@ -106,6 +108,8 @@ class NativeSocketSampler : public Engine {
// Observe the real hook timestamps before recordEvent's running-profiler guard.
// Only gtest binaries contain this seam; normal builds have no extra hook work.
static void (*_record_observer_for_test)(int, u64, u64, ssize_t, u8);
// Replaces the Profiler::recordSample() call inside recordProfilerSample().
static void (*_record_sample_for_test)(ProfiledThread*);
#endif

static NativeSocketSampler* const _instance;
Expand Down Expand Up @@ -225,6 +229,11 @@ class NativeSocketSampler : public Engine {
// Common recording logic shared by all four hooks.
void recordEvent(int fd, u64 t0, u64 t1, ssize_t bytes, u8 op);

// Records the sample under this thread's CriticalSection, or drops it if the
// thread is already inside one: a profiling signal landing while put() holds a
// claimed call-trace slot would otherwise spin forever on its own thread's claim.
static void recordProfilerSample(ProfiledThread* current, u64 bytes, NativeSocketEvent* event);

// Records the event if ret > 0; returns ret unchanged. Shared tail for all four hooks.
static inline ssize_t record_if_positive(int fd, ssize_t ret, u64 t0, u64 t1, u8 op) {
if (ret > 0) {
Expand Down
52 changes: 52 additions & 0 deletions ddprof-lib/src/test/cpp/nativeSocketSampler_ut.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,9 @@
#if defined(__linux__)

#include "nativeSocketSampler.h"
#include "guards.h"
#include "libraryPatcher.h"
#include "threadLocalData.h"

#include <atomic>
#include <chrono>
Expand Down Expand Up @@ -170,8 +172,58 @@ class NativeSocketSamplerTestAccessor {
static void setObserver(void (*observer)(int, u64, u64, ssize_t, u8)) {
NativeSocketSampler::_record_observer_for_test = observer;
}
static void setRecordSampleHook(void (*hook)(ProfiledThread*)) {
NativeSocketSampler::_record_sample_for_test = hook;
}
static void recordProfilerSample(ProfiledThread* current, NativeSocketEvent* event) {
NativeSocketSampler::recordProfilerSample(current, 1, event);
}
};

static int g_recorder_calls;
static bool g_nested_section_entered;

// Stands in for Profiler::recordSample(): a profiling signal on this thread
// would enter CriticalSection the same way before reaching put().
static void record_sample_probe(ProfiledThread* current) {
g_recorder_calls++;
CriticalSection nested(current);
g_nested_section_entered = nested.entered();
}

class NativeSocketSamplerCriticalSectionTest : public ::testing::Test {
protected:
void SetUp() override {
_thread = ProfiledThread::initCurrentThreadSignalSafe();
ASSERT_NE(_thread, nullptr);
g_recorder_calls = 0;
g_nested_section_entered = true;
NativeSocketSamplerTestAccessor::setRecordSampleHook(record_sample_probe);
}
void TearDown() override { NativeSocketSamplerTestAccessor::setRecordSampleHook(nullptr); }

ProfiledThread* _thread = nullptr;
NativeSocketEvent _event;
};

// A signal that lands while the socket sample is in put() must find this
// thread's CriticalSection taken and drop its own sample.
TEST_F(NativeSocketSamplerCriticalSectionTest, RecordsSampleInsideCriticalSection) {
NativeSocketSamplerTestAccessor::recordProfilerSample(_thread, &_event);

EXPECT_EQ(g_recorder_calls, 1);
EXPECT_FALSE(g_nested_section_entered) << "a nested signal could re-enter put()";
}

TEST_F(NativeSocketSamplerCriticalSectionTest, DropsSampleWhenThreadIsInCriticalSection) {
CriticalSection outer(_thread);
ASSERT_TRUE(outer.entered());

NativeSocketSamplerTestAccessor::recordProfilerSample(_thread, &_event);

EXPECT_EQ(g_recorder_calls, 0);
}

static u64 g_io_start;
static u64 g_io_end;
static int g_io_calls;
Expand Down
Loading
Loading