Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
0bd231e
feat(natsock): patch Netty's native epoll transport library by symbol
kaahos Aug 9, 2026
1129d4d
fix: preserve native I/O ownership across recording restarts
kaahos Nov 15, 2026
a8b2b08
Harden Netty socket-image discovery and split its TaskBlock oracles
kaahos Nov 15, 2026
c43caa2
Preserve JDK poll adaptation and retain Netty loader aliases
kaahos Nov 15, 2026
a772790
Merge native I/O custody and cancellation fixes into Netty epoll hooks
kaahos Nov 16, 2026
60b5104
Retry only transient profiler test teardown failures
kaahos Nov 16, 2026
b4ddb30
Isolate lightweight TaskBlock test coordinator waits
kaahos Nov 16, 2026
c856b9f
Hold single Netty hook scopes across profiler restart
kaahos Nov 16, 2026
5dc77d6
Merge crash bootstrap and recorder telemetry into Netty TaskBlocks
kaahos Nov 17, 2026
1e438f8
Merge upstream #663 test follow-ups into #730
kaahos Nov 17, 2026
2a55fc6
Merge upstream exit-order repair into #730
kaahos Nov 18, 2026
62d2964
Merge wall dump and socket lifecycle fixes into #730
kaahos Nov 18, 2026
db7e718
Account for best-effort recorder rejection in Netty tests (#730)
kaahos Nov 18, 2026
5cc687d
Mark unprovisionable musl/OpenJ9 combinations not applicable
kaahos Nov 18, 2026
9552110
test(ci): make the pipeline exclusion literal explicit
kaahos Nov 18, 2026
2a41124
Merge #666 repairs into #730 preserving Netty epoll ownership
kaahos Nov 19, 2026
80f15ce
fix(natsock): skip a failing Netty image instead of disabling socket …
kaahos Nov 19, 2026
c20c521
fix(natsock): key skipped Netty images by base and fingerprint
kaahos Nov 19, 2026
100a752
chore(stack): merge #666 fixes into #730
kaahos Nov 19, 2026
aaa10f4
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
973af00
test(lifecycle): use the renamed transient-failure stop helper
kaahos Nov 20, 2026
cc0e759
test(natsock): drop the Netty marker case for the deleted CodeCache a…
kaahos Nov 20, 2026
3d65832
fix(natsock): leave epoll_pwait2 unhooked when libc lacks it
kaahos Nov 20, 2026
dcaf850
test(natsock): cover Netty skip-table capacity, pin, writable and sta…
kaahos Nov 20, 2026
31c6533
test(natsock): isolate Netty skip-test cases from earlier cases
kaahos Nov 20, 2026
f3177b8
test(natsock): simulate a missing libc original through the scan's ow…
kaahos Nov 20, 2026
7c80e93
docs(natsock): drop the stale single-hook note on the missing-origina…
kaahos Nov 20, 2026
d7b154e
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
ab25d4c
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
cb052bf
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
0965577
fix(test): forward parent heap and ASan placement flags to isolated c…
kaahos Nov 20, 2026
302370d
test(netty): share one recorder-rejection budget across both workers
kaahos Nov 20, 2026
84b9b59
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
03a778e
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
dcf537e
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
1e551de
chore(stack): merge #666 fixes into #730
kaahos Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/scripts/prepare_reports.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ cp ddprof-test/javacore*.txt test-reports/ || true
cp ddprof-test/build/hs_err* test-reports/ || true
cp -r ddprof-lib/build/tmp test-reports/native_build || true
cp -r ddprof-test/build/reports/tests test-reports/tests || true
cp -r ddprof-test/build/netty-epoll test-reports/netty-epoll || true
# The JUnit XML of every attempt, for reading by hand, normally comes from
# flake-evidence/ alone (copied below): run_tests_with_retry.sh snapshots
# each attempt's build/test-results there, and flake-evidence/attempt-<final>
Expand Down
17 changes: 17 additions & 0 deletions .github/scripts/tests/test_prepare_reports.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,5 +48,22 @@ write_xml "$CASE/ddprof-test/build/test-results/testDebug"
|| fail "a populated snapshot must still reach the artifact"
pass "a populated snapshot is shipped as-is"

# Netty's independently launched JVM keeps diagnostics below the build root.
CASE="$TEMP_DIR/case-netty-child"
CHILD="ddprof-test/build/netty-epoll/positive-timeout/attempt-1"
mkdir -p "$CASE/$CHILD" "$CASE/build/logs"
write_xml "$CASE/flake-evidence/attempt-1"
write_xml "$CASE/$CHILD/junit"
printf '%s\n' 'java -Dfixture=netty' > "$CASE/$CHILD/argv.txt"
printf '%s\n' 'child failure witness' > "$CASE/$CHILD/child.log"
printf '%s\n' 'fatal error witness' > "$CASE/$CHILD/hs_err_pid123.log"
( cd "$CASE" && "$SCRIPT" >/dev/null 2>&1 )
for relative in argv.txt child.log hs_err_pid123.log junit/TEST-com.dd.Some.xml; do
cmp "$CASE/$CHILD/$relative" \
"$CASE/test-reports/netty-epoll/positive-timeout/attempt-1/$relative" >/dev/null 2>&1 \
|| fail "Netty child diagnostic missing or changed: $relative"
done
pass "nested Netty child argv, log, crash and JUnit XML are retained"

echo
echo "All $TESTS prepare-reports tests passed."
2 changes: 1 addition & 1 deletion ddprof-lib/src/main/cpp/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ before working on a subsystem.
| Profiling engines (CPU, monotonic wall-clock cadence, allocation, native memory, liveness; prechecks preserve reservoir selections in epoch accounting; timer TID metadata lookup is capped at 256 probes and fails open to ordinary signals; lazy wall-block backing survives recording resets; unowned teardown tails retain slot custody until written, and stop counts the weight of the ones it cannot write as dropped; token-owned completed-at-exit TaskBlocks exclude virtual threads; allocation/liveness stack capture via `allocstack`, with JVMTI retry for unsafe frames, depth-limit truncation and separate verification scratch) | [doc/architecture/ProfilingEngines.md](/doc/architecture/ProfilingEngines.md) |
| JFR integration (FlightRecorder, metadata, constant pools, buffers; bounded DEBUG TaskBlock transition receipts survive recording teardown; wall epochs settle before chunk seals; retained tails carry originating metadata across dumps; stop archives full thread tables before bounded tail-admission retries; output quarantine retains custody and stops producers in non-restartable ERROR) | [doc/architecture/JfrIntegration.md](/doc/architecture/JfrIntegration.md) |
| Live-heap reference chains (implementation, leak-signal gate, scheduling, design) | [doc/architecture/LiveHeapReferenceChainsImplementation.md](/doc/architecture/LiveHeapReferenceChainsImplementation.md), [doc/architecture/ReferenceChainsSignalsExplained.md](/doc/architecture/ReferenceChainsSignalsExplained.md), [doc/ReferenceChainsDesign.md](/doc/ReferenceChainsDesign.md), [doc/ReferenceChainsCollectionSummary.md](/doc/ReferenceChainsCollectionSummary.md) |
| Native I/O hooks (shared-owner JDK import patches, FD classification, marked-thread timeout-probe bypass, cancellation-safe cleanup and synchronous physical-thread IO_WAIT TaskBlocks) | [doc/architecture/NativeIoHooks.md](/doc/architecture/NativeIoHooks.md) |
| Native I/O hooks (shared-owner JDK import patches preserve the JDK 8 poll adapter; base/fingerprint-validated Netty epoll images via `SOCKET_PATCH_NETTY_NATIVE_EPOLL`, FD classification, marked-thread timeout-probe bypass, cancellation-safe cleanup, synchronous physical-thread exit-time IO_WAIT TaskBlocks) | [doc/architecture/NativeIoHooks.md](/doc/architecture/NativeIoHooks.md) |
| Native memory profiling (malloc interception installed per recording and restored on stop, raw-identity exit admission, sampling) | [doc/architecture/NativeMemoryProfiling.md](/doc/architecture/NativeMemoryProfiling.md) |
| glibc malloc arena warning (lock-free arena walk, per-chunk rule, `MallocArenaWarning`/`MallocArenaStats` events, `arenawarn`) | [doc/architecture/MallocArenaWarning.md](/doc/architecture/MallocArenaWarning.md) |
| Startup sanity checks | [doc/architecture/SanityChecks.md](/doc/architecture/SanityChecks.md) |
Expand Down
11 changes: 11 additions & 0 deletions ddprof-lib/src/main/cpp/codeCache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ CodeCache::CodeCache(const char *name, short lib_index,
_plt_offset = 0;
_plt_size = 0;
_debug_symbols = false;
_netty_epoll_markers = 0;

// Initialize build-id fields
_build_id = nullptr;
Expand Down Expand Up @@ -96,6 +97,7 @@ void CodeCache::copyFrom(const CodeCache& other) {
_plt_offset = other._plt_offset;
_plt_size = other._plt_size;
_debug_symbols = other._debug_symbols;
_netty_epoll_markers = other._netty_epoll_markers;

// Copy build-id information
_build_id_len = other._build_id_len;
Expand Down Expand Up @@ -240,6 +242,13 @@ CodeBlob *CodeCache::commitAdd(const void *start, int length,
_blobs[_count]._end = end;
_blobs[_count]._name = prepared.name;
_count++;
if (start != nullptr) {
if (strcmp(prepared.name, "JNI_OnLoad_netty_transport_native_epoll") == 0) {
_netty_epoll_markers |= 1;
} else if (strcmp(prepared.name, "JNI_OnUnload_netty_transport_native_epoll") == 0) {
_netty_epoll_markers |= 2;
}
}
// The cache owns both allocations now; the PreparedAdd destructor must not
// free them.
prepared.name = nullptr;
Expand Down Expand Up @@ -428,6 +437,8 @@ void CodeCache::addImport(void **entry, const char *name) {
saveImport(im_epoll_wait, entry);
} else if (strcmp(name, "epoll_pwait") == 0) {
saveImport(im_epoll_pwait, entry);
} else if (strcmp(name, "epoll_pwait2") == 0) {
saveImport(im_epoll_pwait2, entry);
}
break;
case 'm':
Expand Down
4 changes: 4 additions & 0 deletions ddprof-lib/src/main/cpp/codeCache.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ enum ImportId {
im_ppoll,
im_select,
im_pselect,
im_epoll_pwait2,
NUM_IMPORTS
};

Expand Down Expand Up @@ -193,6 +194,7 @@ class CodeCache {
// LibraryPatcher and MallocHooker protect import writes with different locks.
std::atomic<bool> _imports_patchable{false};
bool _debug_symbols;
u8 _netty_epoll_markers;

FrameDesc *_dwarf_table;
int _dwarf_table_length;
Expand Down Expand Up @@ -279,6 +281,8 @@ class CodeCache {
}

bool hasDebugSymbols() const { return _debug_symbols; }
// Ingested before the native CodeCache is published; discovery never rescans symbols.
bool hasNettyEpollMarkers() const { return _netty_epoll_markers == 3; }

void setDebugSymbols(bool debug_symbols) { _debug_symbols = debug_symbols; }

Expand Down
1 change: 1 addition & 0 deletions ddprof-lib/src/main/cpp/counters.h
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@
X(NATIVE_HOOK_MARK_RESOLVE_FAILED, "native_hook_mark_resolve_failed") \
X(NATIVE_IO_STANDARD_HOOKS_PATCHED, "native_io_standard_hooks_patched") \
X(NATIVE_IO_IBM_BRIDGE_HOOKS_PATCHED, "native_io_ibm_bridge_hooks_patched") \
X(NATIVE_IO_NETTY_IMAGES_SKIPPED, "native_io_netty_images_skipped") \
X(JVMTI_STACKS_FAILED_WRONG_PHASE, "jvmti_stacks_failed_wrong_phase") \
X(JVMTI_STACKS_FAILED_OTHER, "jvmti_stacks_failed_other") \
/* Delegated stacks dropped at slot-lock. Rec-lock drops from all recording \
Expand Down
20 changes: 20 additions & 0 deletions ddprof-lib/src/main/cpp/javaApi.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1596,6 +1596,26 @@ Java_com_datadoghq_profiler_wallclock_WallClockBlockTrackerTestSupport_hasActive
return state != nullptr && state->activeBlockOwner() != BlockRunOwner::NONE
? JNI_TRUE : JNI_FALSE;
}
extern "C" DLLEXPORT jlongArray JNICALL
Java_com_datadoghq_profiler_wallclock_WallClockBlockTrackerTestSupport_nativeOwnerSnapshot0(
JNIEnv* env, jclass unused, jint tid) {
if (ProfiledThread::initCurrentThreadSignalSafe() == nullptr) return nullptr;
ThreadFilter* registry = Profiler::instance()->threadFilter();
ThreadFilter::SlotID slot_id = -1;
// A blocked survivor retains its identity with the preceding recording epoch.
ThreadFilter::Slot* slot = registry->lookupByTid(tid, &slot_id);
if (slot == nullptr) return nullptr;
WallClockBlockTracker::BlockState* state = Profiler::instance()->blockTracker()->slotForId(slot_id);
// Reset-start clears active ownership, but retains identity and token generations.
if (state == nullptr) return nullptr;
jlong values[] = {slot_id, static_cast<jlong>(slot->lifecycleGeneration()),
static_cast<jlong>(slot->recordingEpoch()),
static_cast<jlong>(state->blockGeneration()),
static_cast<jlong>(registry->recordingEpoch())};
jlongArray result = env->NewLongArray(5);
if (result != nullptr) env->SetLongArrayRegion(result, 0, 5, values);
return result;
}
#endif // DEBUG

// LivenessTracker/ReferenceChainTracker test seams. They mutate real tracker
Expand Down
30 changes: 28 additions & 2 deletions ddprof-lib/src/main/cpp/libraryPatcher.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,19 @@ typedef struct _socketPatchedLibrary {
enum SocketPatchTarget : u8 {
SOCKET_PATCH_NONE = 0,
SOCKET_PATCH_STANDARD_JDK_NETWORK,
SOCKET_PATCH_IBM_JCL_BRIDGE
SOCKET_PATCH_IBM_JCL_BRIDGE,
SOCKET_PATCH_NETTY_NATIVE_EPOLL
};

const int SOCKET_BASE_TABLE_SIZE = MAX_NATIVE_LIBS * 2;
// Bounds the linear skip-set scan; a skip beyond it fails the installation closed.
const int SOCKET_SKIPPED_IMAGE_CAPACITY = 64;

// Several append-only caches can describe one live image; base and layout identify it.
typedef struct _socketSkippedImage {
const void* _image_base;
u64 _fingerprint;
} SocketSkippedImage;
// A library to visit with a LibraryPatcher::LiveLibraryWalk
typedef struct _libraryRef {
uintptr_t _base; // lib->imageBase(), the sort key
Expand Down Expand Up @@ -90,6 +99,10 @@ class LibraryPatcher {
static std::vector<SocketPatchEntry> _socket_entries;
static std::vector<SocketPatchedLibrary> _socket_libraries;
static const void* _socket_bases[SOCKET_BASE_TABLE_SIZE];
// Netty images rejected by an image-local failure. Guarded by _lock and cleared
// with the socket patches, so they are retried by the next installation only.
static SocketSkippedImage _socket_skipped_images[SOCKET_SKIPPED_IMAGE_CAPACITY];
static int _socket_skipped_image_count;

// Candidates of the current LiveLibraryWalk, filled by add_live_candidate().
// Guarded by _lock.
Expand Down Expand Up @@ -126,6 +139,11 @@ class LibraryPatcher {
static bool is_profiler_library(CodeCache* lib);
static bool socket_library_patched_unlocked(const void* image_base);
static void remember_socket_library_unlocked(const void* image_base);
static bool socket_image_skipped_unlocked(const CodeCache* lib);
// Both return false only when the skip table is full.
static bool skip_socket_image_unlocked(CodeCache* lib, const char* reason);
static bool skip_socket_image(CodeCache* lib, u64 generation, bool require_active,
const char* reason);
static void unpatch_socket_functions_unlocked(
std::vector<SocketPatchedLibrary>& libraries_to_release);
static void release_socket_libraries(
Expand Down Expand Up @@ -156,10 +174,18 @@ class LibraryPatcher {
using SocketPatchObserverForTest = void (*)(const char* phase);
static void setSocketPatchObserverForTest(SocketPatchObserverForTest observer);
// Called under _lock after lookup; must not reenter the patcher or loader.
// Return true to exercise production's missing-slot rollback.
// Return true to report a missing slot: a Netty image is skipped before any
// write; other images exercise production's missing-slot rollback.
using SocketImportFailureForTest = bool (*)(CodeCache*, ImportId, size_t);
static SocketImportFailureForTest exchangeSocketImportFailureForTest(
SocketImportFailureForTest failure);
// Return true to fail `step` ("pin" outside _lock, "writable" under it) for that image.
using SocketImageFailureForTest = bool (*)(CodeCache*, const char* step);
static SocketImageFailureForTest exchangeSocketImageFailureForTest(
SocketImageFailureForTest failure);
// Treats the libc original of hook_index as missing for later installs; -1 clears.
static int exchangeSocketMissingOriginalForTest(int hook_index);
static void fill_socket_skipped_images_for_test();
static int patch_socket_import_for_test(CodeCache* lib, ImportId import_id,
void* hook, const char* name,
bool retain_library = false);
Expand Down
Loading
Loading