Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
4d67b34
doc: add native-memory profiler exclusion design
jbachorik Nov 12, 2026
8536a6c
refactor: extract LoadedImage to loadedImage_linux.h
jbachorik Nov 12, 2026
84f6185
feat: CodeCache read-only two-slot import accessor
jbachorik Nov 12, 2026
ecb53e6
feat: ELF image admission classifier for conflicting native-memory pr…
jbachorik Nov 12, 2026
79498de
test: native admission fixture DSOs with marker and hook variants
jbachorik Nov 12, 2026
fc7eb7f
test: fixture-DSO classification gtests for the admission marker lookup
jbachorik Nov 13, 2026
1063a58
Fix DT_HASH overwriting DT_GNU_HASH in admission metadata; add synthe…
jbachorik Nov 13, 2026
a805183
Add Linux admission owner walk with loader-iteration lifetime
jbachorik Nov 13, 2026
8ce7ea8
Add shared admission target derivation, combination and error literals
jbachorik Nov 13, 2026
e927ecf
Add platform-neutral admission derivation and combination gtests
jbachorik Nov 13, 2026
0958e1e
Add Linux owner-walk slot and dedup admission gtests
jbachorik Nov 13, 2026
7e7db0b
Route MallocHooker::initialize through the admission check and return…
jbachorik Nov 13, 2026
b519905
Add forked fresh-path admission state-machine gtests
jbachorik Nov 13, 2026
8220d2c
Add forked cached-path, NO_OWNER and concurrent-load admission gtests
jbachorik Nov 13, 2026
a830fc7
Add ddprof stand-in test double with marker exports and GOT rewriting
jbachorik Nov 13, 2026
8d04272
Add child-JVM native-memory admission rejection tests
jbachorik Nov 13, 2026
4c84a56
Add native-memory admission child-JVM lifecycle tests
jbachorik Nov 13, 2026
8776994
Document two-architecture ddprof verification baseline
jbachorik Nov 16, 2026
6755426
Document native-memory admission and its startup limits
jbachorik Nov 16, 2026
c47bf5d
Handle musl test relocation pointers and sanitizer GOT gaps
jbachorik Nov 16, 2026
128f308
Remove generated core dump
jbachorik Nov 17, 2026
fa9d075
Address native memory exclusion review findings
jbachorik Nov 17, 2026
b74edbd
Fix sanitizer detection in admission tests
jbachorik Nov 17, 2026
0609326
Harden native-memory profiler exclusion
jbachorik Nov 19, 2026
78424b7
Harden native-memory admission tests
jbachorik Nov 19, 2026
d51af1e
style: satisfy spotless in NativememDdprofExclusionTest
jbachorik Nov 19, 2026
e1478b3
fix: reject zero bloom_size in GNU-hash admission lookup
jbachorik Nov 19, 2026
030f406
Fix stand-in install comment and restore test SCRIPT_FILE
jbachorik Nov 20, 2026
3952d2f
test(elfparser): mark synthetic PT_LOAD segments readable (PROF-16817)
jbachorik Nov 20, 2026
bc27390
fix(admission): accept a SysV chain terminated exactly at the step cap
jbachorik Nov 20, 2026
54f26e1
Replace GCC 5 overflow builtins in ddprof test stand-in
jbachorik Nov 20, 2026
316e037
Document native-memory admission as implemented
jbachorik Nov 20, 2026
93c951e
Restore runbook index line lost in rebase
jbachorik Nov 20, 2026
327e118
Initialize stand-in forwarders before publishing hooks
jbachorik Nov 20, 2026
242daa7
Merge branch 'main' into dd/there-is-something-weird-going-on-when-bo…
jbachorik Nov 22, 2026
18add5e
test: wait for spin workload readiness before the admission window
jbachorik Nov 22, 2026
95ac270
test: stop the spin workload on every failed or aborted start
jbachorik Nov 22, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ shell-unit-tests:
- bash .gitlab/scripts/tests/includes_test.sh
- bash .gitlab/scripts/tests/test_check_stack_top.sh
- bash .gitlab/scripts/tests/test_generate_benchmarks.sh
- bash ddprof-lib/src/test/resources/native-libs/ddprof-admission/test_makefile_libc_gate.sh
- bash .gitlab/scripts/tests/test_run_gtest_quarantine.sh
- bash .gitlab/jdk-integration/tests/select_test_jdk_test.sh
- bash .gitlab/dd-trace-integration/tests/post_pr_comment_test.sh
Expand Down
150 changes: 141 additions & 9 deletions .gitlab/scripts/tests/test_generate_benchmarks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,34 +67,64 @@ STUB
# run_generator <desc> [KEY=VALUE env assignments...]
# GC_CANCELLED: set => run with CANCELLED=<value>; unset => run with CANCELLED
# removed from the environment. Leaves RESULT (stdout), CHILD (generated
# child pipeline contents) and STACK_INVOKED (whether the stub ran).
# child pipeline contents), STACK_INVOKED and the generator diagnostics.
RUN_N=0
GENERATOR_STATUS=0
GENERATOR_STDERR=""
run_generator() {
local desc="$1"; shift
local dir log
dir=$(setup_sandbox)
RUN_N=$((RUN_N + 1))
log="$WORK/stack_log.$RUN_N"
GENERATOR_STDERR="$WORK/generator_stderr.$RUN_N"
: > "$log"
: > "$GENERATOR_STDERR"
if [ -n "${GC_CANCELLED+x}" ]; then
RESULT=$(cd "$dir" && env CANCELLED="$GC_CANCELLED" STACK_LOG="$log" "$@" \
"$BASH_BIN" "$SCRIPT_FILE" 2>/dev/null)
"$BASH_BIN" "$SCRIPT_FILE" 2>"$GENERATOR_STDERR")
else
RESULT=$(cd "$dir" && env -u CANCELLED STACK_LOG="$log" "$@" \
"$BASH_BIN" "$SCRIPT_FILE" 2>/dev/null)
"$BASH_BIN" "$SCRIPT_FILE" 2>"$GENERATOR_STDERR")
fi
GENERATOR_STATUS=$?
CHILD=$(cat "$dir/generated-benchmarks.yml" 2>/dev/null || true)
STACK_INVOKED=$(cat "$log" 2>/dev/null || true)
}

generator_diagnostics() {
local stderr=""
if [ -n "$GENERATOR_STDERR" ] && [ -f "$GENERATOR_STDERR" ]; then
stderr=$(cat "$GENERATOR_STDERR")
fi
printf 'generator status: %s\nstdout:\n%s\nstderr:\n%s' \
"$GENERATOR_STATUS" "$RESULT" "$stderr"
}

diagnostics_match() {
local captured="$1" status="$2" stdout="$3" stderr="$4" expected
expected=$(printf 'generator status: %s\nstdout:\n%s\nstderr:\n%s' "$status" "$stdout" "$stderr")
[[ "$captured" == *"$expected" ]]
}

expect_result() {
local desc="$1" expected="$2" actual
# the decision contract is the single RUN_BENCHMARKS= line on stdout
actual=$(printf '%s\n' "$RESULT" | sed -n 's/^RUN_BENCHMARKS=//p' | head -1)
local desc="$1" expected="$2" actual count
if [ "$GENERATOR_STATUS" -ne 0 ]; then
fail "$desc — generator exited with status $GENERATOR_STATUS"$'\n'"$(generator_diagnostics)"
return
fi

count=$(printf '%s\n' "$RESULT" | awk '/^RUN_BENCHMARKS=/{n++} END{print n+0}')
if [ "$count" -ne 1 ]; then
fail "$desc — expected exactly one RUN_BENCHMARKS= line, got $count"$'\n'"$(generator_diagnostics)"
return
fi

actual=$(printf '%s\n' "$RESULT" | sed -n 's/^RUN_BENCHMARKS=//p')
if [ "$actual" = "$expected" ]; then
pass "$desc"
else
fail "$desc — expected RUN_BENCHMARKS=$expected, got '${RESULT}'"
fail "$desc — expected RUN_BENCHMARKS=$expected"$'\n'"$(generator_diagnostics)"
fi
}

Expand All @@ -105,14 +135,14 @@ expect_child() {
if printf '%s' "$CHILD" | grep -q 'skip-benchmarks:'; then
pass "$desc"
else
fail "$desc — expected the noop child, got: ${CHILD:0:80}"
fail "$desc — expected the noop child, got: ${CHILD:0:80}"$'\n'"$(generator_diagnostics)"
fi
;;
real)
if [ "$CHILD" = "real-child-pipeline" ]; then
pass "$desc"
else
fail "$desc — expected the real child pipeline, got: ${CHILD:0:80}"
fail "$desc — expected the real child pipeline, got: ${CHILD:0:80}"$'\n'"$(generator_diagnostics)"
fi
;;
esac
Expand All @@ -128,6 +158,77 @@ expect_stack_check() {
fi
}

assert_result_rejected_with_diagnostics() {
local desc="$1" output="$2" status="$3" stderr="$4"
local stderr_file="$WORK/assertion.stderr" captured
printf '%s' "$stderr" > "$stderr_file"
if captured=$(
FAILED=0
RESULT="$output"
GENERATOR_STATUS="$status"
GENERATOR_STDERR="$stderr_file"
expect_result "$desc" true
[ "$FAILED" -eq 1 ]
) 2>&1; then
if diagnostics_match "$captured" "$status" "$output" "$stderr"; then
pass "$desc rejects output and reports complete diagnostics"
else
fail "$desc did not report status, full stdout, and stderr"
fi
else
fail "$desc was not rejected"
fi
}

assert_run_generator_captures_exit_status() {
local script="$WORK/nonzero-generator.sh" saved_script_file captured
saved_script_file="$SCRIPT_FILE"
cat > "$script" << 'STUB'
printf '%s\n' 'RUN_BENCHMARKS=true'
printf '%s\n' 'nonzero-process-stderr' >&2
exit 17
STUB
if captured=$(
FAILED=0
SCRIPT_FILE="$script"
GC_CANCELLED=true
run_generator "nonzero process exit"
expect_result "nonzero process exit" true
[ "$GENERATOR_STATUS" -eq 17 ] && [ "$FAILED" -eq 1 ]
) 2>&1; then
if diagnostics_match "$captured" 17 'RUN_BENCHMARKS=true' 'nonzero-process-stderr'; then
pass "run_generator captures nonzero exit status and diagnostics"
else
fail "run_generator did not report the nonzero process status and diagnostics"
fi
else
fail "run_generator did not reject the nonzero process"
fi
SCRIPT_FILE="$saved_script_file"
}

assert_child_rejected_with_diagnostics() {
local stderr_file="$WORK/child-assertion.stderr" captured
printf '%s' 'child-stderr-marker' > "$stderr_file"
if captured=$(
FAILED=0
RESULT='child stdout marker'
GENERATOR_STATUS=17
GENERATOR_STDERR="$stderr_file"
CHILD='unexpected child'
expect_child 'controlled child failure' noop
[ "$FAILED" -eq 1 ]
) 2>&1; then
if diagnostics_match "$captured" 17 'child stdout marker' 'child-stderr-marker'; then
pass "child failure reports status, full stdout, and stderr"
else
fail "child failure did not report complete generator diagnostics"
fi
else
fail "controlled child assertion was not rejected"
fi
}

# GC_CANCELLED is set and unset as plain statements, never as `VAR=x func`
# prefixes: whether such an assignment outlives the function is
# shell-version-dependent, so each case must control its own mode explicitly.
Expand Down Expand Up @@ -164,6 +265,37 @@ run_generator "garbled" FAKE_STACK_OUT=""
expect_result "garbled stack-check output fails open" true
expect_child "garbled stack-check output emits the real child" real

# Exercise parser failures and diagnostics without altering the extracted
# generator scenarios above.
assert_result_rejected_with_diagnostics "missing contract line" \
'generator stdout without a decision' 0 'missing-contract-stderr'
assert_result_rejected_with_diagnostics "duplicate contract lines" \
$'RUN_BENCHMARKS=true\nRUN_BENCHMARKS=false' 0 'duplicate-contract-stderr'
assert_result_rejected_with_diagnostics "wrong contract value" \
'RUN_BENCHMARKS=false' 0 'wrong-value-stderr'
assert_result_rejected_with_diagnostics "nonzero generator status" \
'RUN_BENCHMARKS=true' 17 'nonzero-status-stderr'
assert_run_generator_captures_exit_status
assert_child_rejected_with_diagnostics

saved_script_file="$SCRIPT_FILE"
SCRIPT_FILE="$WORK/stream-separation-generator.sh"
cat > "$SCRIPT_FILE" << 'STUB'
printf '%s\n' 'RUN_BENCHMARKS=true'
printf '%s\n' 'stderr-only-marker' >&2
printf '%s\n' 'real-child-pipeline' > generated-benchmarks.yml
STUB
GC_CANCELLED=true
run_generator "stream-separation"
if [ "$RESULT" = 'RUN_BENCHMARKS=true' ] && \
[ "$(cat "$GENERATOR_STDERR")" = 'stderr-only-marker' ]; then
pass "generator stderr stays separate from contract stdout"
else
fail "generator stderr contaminated stdout or was not captured"$'\n'"$(generator_diagnostics)"
fi
expect_result "controlled generator emits one contract line" true
SCRIPT_FILE="$saved_script_file"

if [ "$FAILED" -ne 0 ]; then
echo "test_generate_benchmarks: FAILED"
exit 1
Expand Down
2 changes: 1 addition & 1 deletion ddprof-lib/src/main/cpp/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ before working on a subsystem.
| Profiling engines (CPU, wall-clock, allocation, native memory, liveness; allocation/liveness stack capture via `allocstack`, with JVMTI retry for unsafe frames, depth-limit truncation and separate verification scratch) | [doc/architecture/ProfilingEngines.md](/doc/architecture/ProfilingEngines.md) |
| JFR integration (FlightRecorder, metadata, constant pools, buffers) | [doc/architecture/JfrIntegration.md](/doc/architecture/JfrIntegration.md) |
| Live-heap reference chains (implementation, leak-signal gate, scheduling, design) | [doc/architecture/LiveHeapReferenceChainsImplementation.md](/doc/architecture/LiveHeapReferenceChainsImplementation.md), [doc/architecture/ReferenceChainsSignalsExplained.md](/doc/architecture/ReferenceChainsSignalsExplained.md), [doc/ReferenceChainsDesign.md](/doc/ReferenceChainsDesign.md), [doc/ReferenceChainsCollectionSummary.md](/doc/ReferenceChainsCollectionSummary.md) |
| Native memory profiling (malloc interception installed per recording and restored on stop, sampling) | [doc/architecture/NativeMemoryProfiling.md](/doc/architecture/NativeMemoryProfiling.md) |
| Native memory profiling (malloc interception installed per recording and restored on stop; startup admission check for conflicting native-memory profilers; sampling) | [doc/architecture/NativeMemoryProfiling.md](/doc/architecture/NativeMemoryProfiling.md) |
| glibc malloc arena warning (lock-free arena walk, per-chunk rule, `MallocArenaWarning`/`MallocArenaStats` events, `arenawarn`) | [doc/architecture/MallocArenaWarning.md](/doc/architecture/MallocArenaWarning.md) |
| Startup sanity checks | [doc/architecture/SanityChecks.md](/doc/architecture/SanityChecks.md) |
| SIGSEGV-safe library patching | [doc/architecture/SigsegvPatching.md](/doc/architecture/SigsegvPatching.md) |
Expand Down
12 changes: 12 additions & 0 deletions ddprof-lib/src/main/cpp/codeCache.h
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,11 @@ class CodeCache {
// Like findImport(), but never makes the GOT writable, so it does not touch
// the library's memory. For bookkeeping only; patch through findImport().
void **peekImport(ImportId id) const { return _imports[id][PRIMARY]; }
// Read-only access to a single slot cell: returns the stored GOT-entry
// pointer for the given import and slot, or NULL when nothing was saved
// there. Never makes the GOT writable and never touches the library's
// memory. Reads PRIMARY or SECONDARY; for bookkeeping only.
void **peekImport(ImportId id, ImportType type) const { return _imports[id][type]; }
// Makes the import slots writable even if they were made so before: when the
// library was unloaded and reloaded at the same address, the new mapping's
// GOT is read-only again under full RELRO.
Expand All @@ -365,6 +370,13 @@ class CodeCache {
}
void patchImport(ImportId, void *hook_func);

#ifdef UNIT_TEST
// Whether makeImportsWritable() has already switched this cache's import
// slots to writable. Read-only observation for tests; compiled only into
// gtest binaries.
bool importsPatchableForTest() const { return _imports_patchable; }
#endif

CodeBlob *findBlob(const char *name);
CodeBlob *findBlobByAddress(const void *address);
const void *binarySearch(const void *address, const char **name);
Expand Down
94 changes: 94 additions & 0 deletions ddprof-lib/src/main/cpp/loadedImage_linux.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright 2026, Datadog, Inc.
* SPDX-License-Identifier: Apache-2.0
*/

// Linux-only: ELF image views built from program-header tables. Include only
// from _linux translation units; there is no non-Linux counterpart.
#ifdef __linux__

#ifndef _LOADED_IMAGE_LINUX_H
#define _LOADED_IMAGE_LINUX_H

#include <stdint.h>

#include <elf.h>

#include "os.h"

typedef Elf64_Phdr ElfProgramHeader;

// A loaded ELF image, described by its program headers. Only the file-backed
// parts of its PT_LOAD segments are known to be mapped.
class LoadedImage {
private:
const ElfProgramHeader* _ph;
int _phnum;
uintptr_t _load_bias;

bool minLoadVaddr(uintptr_t& out) const {
uintptr_t min_vaddr = UINTPTR_MAX;
for (int i = 0; i < _phnum; i++) {
if (_ph[i].p_type == PT_LOAD && _ph[i].p_vaddr < min_vaddr) min_vaddr = _ph[i].p_vaddr;
}
if (min_vaddr == UINTPTR_MAX) return false;
out = min_vaddr;
return true;
}

public:
LoadedImage(const ElfProgramHeader* ph, int phnum, uintptr_t load_bias)
: _ph(ph), _phnum(phnum), _load_bias(load_bias) {}

bool address(uintptr_t vaddr, uintptr_t& out) const {
return !__builtin_add_overflow(_load_bias, vaddr, &out);
}

bool pageBase(uintptr_t& out) const {
uintptr_t min_vaddr, addr;
if (!minLoadVaddr(min_vaddr) || !address(min_vaddr, addr)) return false;
// Add before masking to preserve the loader's page-base calculation.
out = addr & ~OS::page_mask;
return true;
}

// True if [addr, addr + size) lies inside a readable, file-backed PT_LOAD.
bool readable(uintptr_t addr, size_t size) const {
for (int i = 0; i < _phnum; i++) {
if (_ph[i].p_type != PT_LOAD || (_ph[i].p_flags & PF_R) == 0) continue;
uintptr_t start;
if (!address(_ph[i].p_vaddr, start)) continue;
if (addr >= start && addr - start <= _ph[i].p_filesz
&& size <= _ph[i].p_filesz - (addr - start)) {
return true;
}
}
return false;
}

const ElfProgramHeader* find(uint32_t type) const {
for (int i = 0; i < _phnum; i++) {
if (_ph[i].p_type == type) return &_ph[i];
}
return NULL;
}

// A pointer from the dynamic section. glibc relocates them in place, musl
// and the vDSO do not; an unrelocated one is a vaddr below the image. The
// same test as ElfParser::dyn_ptr().
bool dynamicPointer(uintptr_t d_ptr, uintptr_t& out) const {
uintptr_t min_vaddr;
if (!minLoadVaddr(min_vaddr)) return false;

uintptr_t relocation_threshold;
if (!address(min_vaddr & ~OS::page_mask, relocation_threshold)) return false;
if (d_ptr < relocation_threshold) return address(d_ptr, out);

out = d_ptr;
return true;
}
};

#endif // _LOADED_IMAGE_LINUX_H

#endif // __linux__
Loading
Loading