From 908bb2e5106f9138968ba40b4bb5e59bdb5b6f72 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:27 +0200 Subject: [PATCH 01/27] feat(memtrack): capture allocation stacks in eBPF Copy the caller's user stack in chunks at allocator entry and fold an FNV-1a digest over it in the kernel. The digest rides on the allocation event as stack_hash; the copied bytes, a DWARF-numbered register snapshot and a frame-pointer walk are emitted once per distinct digest on a dedicated ring buffer, so unwinding and symbolication can happen offline. Capture stays off until userspace sets the rodata toggle, so allocator probes are unchanged by default. Refs COD-3222 --- crates/memtrack/src/ebpf/c/allocator.h | 25 +- crates/memtrack/src/ebpf/c/event.h | 57 +++- crates/memtrack/src/ebpf/c/main.bpf.c | 1 + .../memtrack/src/ebpf/c/stack_capture.bpf.h | 275 ++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 21 +- 5 files changed, 360 insertions(+), 19 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/stack_capture.bpf.h diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc238..8de96317 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c9..f4b33c75 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,48 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. The scratch buffer holding + * header plus bytes is a per-CPU map value, capped at PCPU_MIN_UNIT_SIZE + * (32 KiB) by the kernel allocator. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024 - 512) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_PREEMPTED 5 +#define MEMTRACK_STACK_COUNTER_COUNT 6 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +71,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff0..b405f572 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 00000000..28f7fcf8 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,275 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* At allocator entry the caller's raw user stack is copied and hashed; the hash + * travels on the allocation event as its stack identity. The first time a hash + * is seen, the copied bytes plus a register snapshot are emitted as a stack + * record so the stack can be DWARF-unwound offline, with an in-kernel + * frame-pointer walk alongside as the fallback. + * + * Hashing raw bytes rather than unwound frames splits one call path into + * several identities whenever locals or arguments in the copied region differ. + * That only costs extra records; the reverse trade (aliasing) would corrupt + * attribution. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_size = 8192; + +#define STACK_TRACE_MAX_DEPTH 127 +/* Copy granularity: a recovered length is exact only to within one chunk. */ +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +/* Frame-pointer walk results, indexed by the id bpf_get_stackid() returns. */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* Records are bulky but rare (one per distinct hash), so they get their own + * ring rather than inflating the fixed-size `struct event` path. */ +BPF_RINGBUF(stacks, 64 * 1024 * 1024); +BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +/* The record is built in place here and handed to the ring buffer as one + * contiguous variable-length blob. `words` aliases `bytes` so the hash loop + * reads whole registers without a per-byte shift chain. */ +struct stack_scratch_buf { + struct stack_header header; + union { + __u8 bytes[MEMTRACK_MAX_STACK_COPY]; + __u64 words[MEMTRACK_MAX_STACK_COPY / 8]; + }; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct stack_scratch_buf); +} stack_scratch SEC(".maps"); + +/* The value is 64-bit because the BPF backend of older clang cannot select a + * 32-bit atomic compare-and-swap. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} stack_busy SEC(".maps"); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +/* Returns the stack identity, or 0 when nothing could be copied. */ +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + __u32 zero = 0; + struct stack_scratch_buf* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); + if (!scratch) { + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + __u32 want = stack_copy_size; + if (want > MEMTRACK_MAX_STACK_COPY) { + want = MEMTRACK_MAX_STACK_COPY; + } + want &= ~(__u32)(STACK_COPY_CHUNK - 1); + if (want < STACK_COPY_CHUNK) { + want = STACK_COPY_CHUNK; + } + + /* bpf_probe_read_user() is all-or-nothing and the readable region ends at + * the top of the stack mapping, which is not knowable up front, so the copy + * advances in chunks and stops at the first unreadable one. + * + * Each chunk is hashed as it lands, over a constant iteration count the + * compiler fully unrolls. One loop over the whole copy instead costs the + * verifier a state fork per word and blows the one-million instruction + * budget well below the maximum copy size. */ + __u64 hash = FNV64_OFFSET; + __u32 got = 0; +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= MEMTRACK_MAX_STACK_COPY; + off += STACK_COPY_CHUNK) { + if (off >= want) { + break; + } + if (bpf_probe_read_user(&scratch->bytes[off], STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + __u32 base = off >> 3; +#pragma unroll + for (__u32 word = 0; word < STACK_COPY_CHUNK / 8; word++) { + hash = (hash ^ scratch->words[base + word]) * FNV64_PRIME; + } + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= want; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + /* Fold in the length so a truncated prefix of a deep stack cannot collide + * with a full copy of a shallower one, and keep 0 reserved as the "no + * stack" marker on allocation events. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST: already emitted */ + return hash; + } + if (gate_result != 0) { + /* A full gate cannot retain this identity, so emit it on every + * occurrence rather than make the allocation hash unresolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + scratch->header.hash = hash; + scratch->header.timestamp = bpf_ktime_get_ns(); + scratch->header.stackid = stackid; + scratch->header.sp = sp; + scratch->header.pid = ids.tgid; + scratch->header.tid = ids.tid; + scratch->header.copy_len = got; + scratch->header.truncated = truncated; + scratch->header._pad[0] = 0; + scratch->header._pad[1] = 0; + scratch->header._pad[2] = 0; + fill_stack_regs(&scratch->header.regs, ctx); + + if (bpf_ringbuf_output(&stacks, scratch, sizeof(struct stack_header) + got, 0) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + bpf_map_delete_elem(&seen_stack_hashes, &hash); + } + + return hash; +} + +/* Copy and hash the caller's stack, emitting a record on first sight of the + * resulting identity. Returns 0 when capture is off or nothing was copied. */ +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + __u32 zero = 0; + __u64* busy = bpf_map_lookup_elem(&stack_busy, &zero); + if (!busy) { + return 0; + } + /* uprobe_multi runs programs without the bpf_prog_active recursion guard, + * so a task preempting this one on the same CPU could corrupt the scratch. */ + if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); + return 0; + } + + __u64 hash = capture_stack_inner(ctx, ids); + + /* A plain store, not an atomic release: the only contender is a task that + * preempted this one on this same CPU, and the context switch between them + * already orders the write. The BPF backend cannot select a release store. */ + *busy = 0; + return hash; +} + +/* Hand an identity to the matching uretprobe. */ +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +/* The identity stashed by the matching entry probe, or 0 when capture is off or + * the entry probe bailed out. The slot is per-thread but shared by allocators, + * so every return path must clear it. */ +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9..ca53593d 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } From 09b0112b95919acf6de082f211259baa9bf98706 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:53 +0200 Subject: [PATCH 02/27] feat(memtrack): add userspace stack-capture module Add the userspace half of allocation stack capture: env-driven configuration, stack-definition ring parsing, loss counters, per-pid module mapping tracking, a folding recorder that deduplicates definitions and counts occurrences, and the report it produces. Nothing constructs these yet; the tracker wiring follows. Refs COD-3222 --- Cargo.lock | 11 ++ crates/memtrack/src/ebpf/events.rs | 23 ++- crates/memtrack/src/ebpf/mod.rs | 3 + crates/memtrack/src/ebpf/stacks/config.rs | 54 +++++++ crates/memtrack/src/ebpf/stacks/counters.rs | 38 +++++ crates/memtrack/src/ebpf/stacks/events.rs | 143 ++++++++++++++++++ crates/memtrack/src/ebpf/stacks/mod.rs | 3 + crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/dlopen_tests.rs | 11 +- crates/memtrack/tests/shared.rs | 7 +- crates/runner-shared/Cargo.toml | 1 + .../runner-shared/benches/memtrack_writer.rs | 37 +++-- .../src/artifacts/memtrack/mod.rs | 93 ++++++++++-- .../src/artifacts/memtrack/pipeline.rs | 5 +- 14 files changed, 396 insertions(+), 35 deletions(-) create mode 100644 crates/memtrack/src/ebpf/stacks/config.rs create mode 100644 crates/memtrack/src/ebpf/stacks/counters.rs create mode 100644 crates/memtrack/src/ebpf/stacks/events.rs create mode 100644 crates/memtrack/src/ebpf/stacks/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 1acec08f..8e555476 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3616,6 +3616,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4062,6 +4063,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a5..79597357 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -34,13 +34,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +55,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -157,6 +166,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +178,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +201,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +213,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 2aa96549..d964ebed 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,12 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; +pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs new file mode 100644 index 00000000..dd99b8c7 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -0,0 +1,54 @@ +use crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY; +use crate::prelude::*; + +pub const DEFAULT_STACK_COPY_SIZE: u32 = 8192; + +/// The per-allocation stack copy budget, or `None` when capture was explicitly +/// disabled with `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`. Capture is on by default. +pub fn stack_copy_size_from_env() -> Option { + if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref() == Ok("0") { + return None; + } + + let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { + Ok(value) => match value.parse::() { + Ok(size) => size, + Err(error) => { + warn!( + "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" + ); + DEFAULT_STACK_COPY_SIZE + } + }, + Err(_) => DEFAULT_STACK_COPY_SIZE, + }; + + Some(clamp_copy_size(copy_size)) +} + +/// The kernel copies whole chunks, so a budget that is not a multiple of one +/// would hash bytes it never emits. +pub fn clamp_copy_size(copy_size: u32) -> u32 { + const CHUNK: u32 = 512; + (copy_size / CHUNK * CHUNK).clamp(CHUNK, MEMTRACK_MAX_STACK_COPY) +} + +#[cfg(test)] +mod tests { + use super::clamp_copy_size; + + #[test] + fn rounds_down_to_a_whole_chunk() { + assert_eq!(clamp_copy_size(8_700), 8_192); + } + + #[test] + fn clamps_to_low_bound() { + assert_eq!(clamp_copy_size(63), 512); + } + + #[test] + fn clamps_to_high_bound() { + assert_eq!(clamp_copy_size(u32::MAX), 32_256); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/counters.rs b/crates/memtrack/src/ebpf/stacks/counters.rs new file mode 100644 index 00000000..6e6d76b1 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/counters.rs @@ -0,0 +1,38 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, + /// Captures skipped because another BPF program used the per-CPU scratch. + pub preempted: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + preempted: slot(map, MEMTRACK_STACK_COUNTER_PREEMPTED)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs new file mode 100644 index 00000000..5d1b6fbf --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/events.rs @@ -0,0 +1,143 @@ +use crate::ebpf::events::bindings::stack_header; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; + +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs new file mode 100644 index 00000000..fc5eee33 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod counters; +pub mod events; diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index db5fe643..2d33dccf 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -110,7 +110,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index c9e1c626..342cbf6b 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -31,7 +31,7 @@ macro_rules! assert_events_snapshot { matches!( e.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -108,7 +108,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -124,6 +124,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + ) }) .sorted_by_key(|e| e.timestamp) @@ -281,7 +282,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab9..9c3c9f18 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8..3a432866 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -14,14 +14,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -90,12 +100,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +121,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -150,7 +167,7 @@ fn encode_events_realistic(bencher: Bencher, n_workers: usize) { bencher.bench_local(|| { let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); output }); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c6..433aab5a 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +102,31 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + + Stack { + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +156,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -167,21 +206,47 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +255,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +280,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +333,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed..8cac46f0 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } From 5ac0d19678fe18019523144058ec3509767988c8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 11:33:08 +0200 Subject: [PATCH 03/27] feat(memtrack): enable stack capture through the tracker Wire the capture rodata and map sizing into skeleton load, poll the stack-definition ring alongside the event ring, and expose the loss counters and frame-pointer chains. The attach worker snapshots module mappings while it holds a process stopped, which is the only point they are guaranteed readable. Guard the lifecycle: finishing with a live session would block forever on the recorder, and a second spawn would leave the capture rings undrained, so both now fail with a descriptive error. With capture disabled the ring buffer and frame-pointer map shrink to the allocator minimum rather than reserving tens of MiB. Refs COD-3222 --- crates/memtrack/src/ebpf/memtrack/maps.rs | 5 + crates/memtrack/src/ebpf/memtrack/mod.rs | 100 ++++++++++++--- crates/memtrack/src/ebpf/memtrack/teardown.rs | 117 ++++++++++++++++++ crates/memtrack/src/ebpf/tracker.rs | 47 ++++++- crates/memtrack/src/session.rs | 3 + 5 files changed, 247 insertions(+), 25 deletions(-) create mode 100644 crates/memtrack/src/ebpf/memtrack/teardown.rs diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d46..0a60f270 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -68,6 +69,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 8586872e..df8505e3 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -20,11 +20,14 @@ mod macros; mod allocator; mod maps; mod rmap; +mod teardown; mod tracking; pub use maps::OwnershipMaps; pub use rmap::RmapSupport; +use teardown::FdHolder; + use crate::bpf_token::has_delegated_bpf_token; /// Which attach mechanism a loaded skeleton uses for its uprobes. See @@ -123,19 +126,25 @@ pub struct MemtrackBpf { impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool) -> Result { + pub fn new_with_rmap(track_rmap: bool, stack_copy_size: Option) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap) + Self::with_variant(variant, track_rmap, stack_copy_size) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + /// + /// `stack_copy_size` turns on allocation stack capture. + pub fn with_variant( + variant: BpfVariant, + track_rmap: bool, + stack_copy_size: Option, + ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -164,6 +173,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if let Some(copy_size) = stack_copy_size { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_size = copy_size; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if stack_copy_size.is_none() { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so fentries whose targets @@ -227,6 +249,38 @@ impl MemtrackBpf { )) } + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::stacks::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + Some(event) + }; + + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stacks, + parse, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( @@ -247,28 +301,36 @@ impl MemtrackBpf { self.probes.len() } - /// Detach all BPF links in parallel. Closing a uprobe link blocks on two - /// RCU grace periods in the kernel, but concurrent waiters share grace - /// periods, so closing from many threads scales near-linearly. + /// Detach all BPF links without waiting for the kernel to complete it. + /// + /// Each release waits for a tasks-trace RCU grace period, and that wait is + /// unbounded: a kernel that has stopped completing them (a BPF program that + /// oopsed leaves a reader that never exits) blocks the caller forever, with + /// no way out even by exiting, since exiting closes the same descriptors. A + /// holder process takes them over, so the releases happen off the critical + /// path and the run always finishes with the trace it has. + /// + /// The probes stay installed until the holder is done, so a process starting + /// right after this returns can still trap into them. pub fn detach_probes(&mut self) { - const DETACH_THREADS: usize = 32; - - let mut probes = std::mem::take(&mut self.probes); + let probes = std::mem::take(&mut self.probes); if probes.is_empty() { return; } - debug!("Detaching {} BPF links", probes.len()); - let start = std::time::Instant::now(); - let chunk_size = probes.len().div_ceil(DETACH_THREADS); - std::thread::scope(|scope| { - while !probes.is_empty() { - let split_at = probes.len().saturating_sub(chunk_size); - let chunk = probes.split_off(split_at); - scope.spawn(move || drop(chunk)); + let count = probes.len(); + let holder = FdHolder::fork() + .inspect_err(|e| warn!("Detaching without a descriptor holder: {e:#}")) + .ok(); + drop(probes); + + match holder { + Some(holder) => { + debug!("Releasing {count} BPF links in holder {}", holder.pid()); + holder.release(); } - }); - debug!("Detached BPF links in {:?}", start.elapsed()); + None => debug!("Detached {count} BPF links"), + } } } diff --git a/crates/memtrack/src/ebpf/memtrack/teardown.rs b/crates/memtrack/src/ebpf/memtrack/teardown.rs new file mode 100644 index 00000000..2214ba97 --- /dev/null +++ b/crates/memtrack/src/ebpf/memtrack/teardown.rs @@ -0,0 +1,117 @@ +use crate::prelude::*; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +/// A forked process that keeps a second reference to every descriptor this +/// process holds, so releasing them here costs nothing. +/// +/// Releasing a classic uprobe link runs `perf_event_detach_bpf_prog()`, which +/// waits for a tasks-trace RCU grace period. That wait is unbounded: a kernel +/// whose grace periods have stalled never completes it, and the process cannot +/// escape it by exiting either, since exiting closes the very same descriptors. +/// A descriptor closed while another process still holds it only drops a +/// refcount, so with a holder alive the release — and the wait — happens in the +/// holder instead, off the critical path of whoever forked it. +pub(super) struct FdHolder { + pid: libc::pid_t, + /// Closing this releases the holder from its wait. + release: OwnedFd, +} + +impl FdHolder { + /// Fork a holder for the descriptors currently open. Descriptors opened + /// afterwards are not covered. + pub(super) fn fork() -> Result { + let (read_end, write_end) = pipe()?; + + // SAFETY: the child touches nothing but the pipe and _exit(), so it + // cannot deadlock on a lock a thread of the parent held across the fork. + let pid = unsafe { libc::fork() }; + ensure!( + pid >= 0, + "fork() failed: {}", + std::io::Error::last_os_error() + ); + + if pid == 0 { + unsafe { hold_until_released(read_end.as_raw_fd(), write_end.as_raw_fd()) }; + } + + Ok(Self { + pid, + release: write_end, + }) + } + + /// Process id of the holder, for logging. + pub(super) fn pid(&self) -> libc::pid_t { + self.pid + } + + /// Let the holder release the descriptors. It is not waited for: it finishes + /// whenever the kernel lets it and is reaped by init. + pub(super) fn release(self) { + drop(self.release); + } +} + +fn pipe() -> Result<(OwnedFd, OwnedFd)> { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: pipe() writes two descriptors into `fds`. + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + ensure!( + rc == 0, + "pipe() failed: {}", + std::io::Error::last_os_error() + ); + // SAFETY: both descriptors are fresh and owned by this process. + unsafe { Ok((OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1]))) } +} + +/// Block until the write end is closed, then exit, which closes every inherited +/// descriptor and performs whatever release work the last close of each entails. +/// +/// # Safety +/// +/// Only for the child of a `fork()`: it never returns, and calls nothing that is +/// unsafe to call between `fork()` and `_exit()` in a multi-threaded process. +unsafe fn hold_until_released(read_end: libc::c_int, write_end: libc::c_int) -> ! { + unsafe { + libc::close(write_end); + + let mut byte = 0u8; + loop { + let rc = libc::read(read_end, (&raw mut byte).cast(), 1); + if rc >= 0 || *libc::__errno_location() != libc::EINTR { + break; + } + } + + libc::_exit(0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The holder must outlive its parent's closes and exit only once released: + /// exiting early would make the parent's own close the last one and put the + /// unbounded wait back on the caller. + #[test] + fn holder_exits_only_once_released() { + let holder = FdHolder::fork().unwrap(); + let pid = holder.pid(); + + std::thread::sleep(std::time::Duration::from_millis(50)); + let mut status = 0; + // SAFETY: waitpid() writes only into `status`. + let reaped = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(reaped, 0, "holder exited before being released"); + + holder.release(); + // SAFETY: waitpid() writes only into `status`. It holds no descriptor + // whose release can block, so it exits as soon as it is released. + let reaped = unsafe { libc::waitpid(pid, &mut status, 0) }; + assert_eq!(reaped, pid, "holder did not exit after being released"); + } +} diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index dc9bc13e..5536e532 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,5 +1,7 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; @@ -7,6 +9,7 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; @@ -19,6 +22,10 @@ pub struct TrackerOptions { /// Reconstruct per-process RSS from the folio rmap fentry hooks. #[builder(default = false)] pub rmap: bool, + /// Bytes of user stack to copy for each allocation event. `None` leaves + /// stack capture off; values are clamped to the supported range. + #[builder(default = None)] + pub stack_copy_size: Option, } impl TrackerOptions { @@ -29,6 +36,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) + .stack_copy_size(stack_copy_size_from_env()) .build() } } @@ -37,6 +45,9 @@ pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, + /// The dedup gate spans the whole BPF object, so a second session would + /// reference stack records the first one already consumed. + stacks_polled: Option, } impl Tracker { @@ -48,9 +59,11 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { + let copy_size = options.stack_copy_size.map(clamp_copy_size); Self::build( - MemtrackBpf::new_with_rmap(options.rmap)?, + MemtrackBpf::new_with_rmap(options.rmap, copy_size)?, options.allocators, + copy_size.is_some(), ) } @@ -58,13 +71,17 @@ impl Tracker { /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; - Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) + Self::build( + MemtrackBpf::with_variant(variant, track_rmap, None)?, + true, + false, + ) } /// Build a tracker: attach lifetime tracepoints (and rmap fentries when the /// skeleton was opened for them), plus, when `allocators` is set, the /// exec-mapping watcher and the on-demand allocator-attach worker. - fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { + fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; @@ -83,6 +100,7 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators, + stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), }) } @@ -95,6 +113,14 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = match &self.stacks_polled { + Some(polled) if polled.swap(true, Ordering::Relaxed) => { + bail!("stack capture supports a single spawned command per tracker") + } + Some(_) => true, + None => false, + }; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -102,6 +128,7 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; + match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -110,14 +137,17 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let poller = { + let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx)?, stack_poller) }; resume(pid)?; - Ok(Session::new(child, rx, poller)) + Ok(Session::new(child, rx, poller, stack_poller)) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -138,6 +168,11 @@ impl Tracker { self.bpf.lock().dropped_events_count() } + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33fe..9bed66b7 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -10,6 +10,7 @@ pub struct Session { child: Child, events: Option>, _poller: RingBufferPoller, + _stack_poller: Option, } impl Session { @@ -17,11 +18,13 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, } } From 7bf49b8582dc7a5dd04c991b310f81562a7157ea Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:01:18 +0200 Subject: [PATCH 04/27] test(memtrack): cover allocation stack capture Add a fixture with two non-inlinable malloc call paths and privileged tests over it: distinct call paths get distinct identities with module mappings for the binary and libc, repeated calls deduplicate, and the default-off path still reports allocations. Two cases guard failure modes the default budget cannot reach. The maximum copy budget is the only configuration that exercises the verifier's instruction limit, since the frozen rodata makes the copy and hash loops scale with the configured size. Shrinking the frame-pointer map to one slot proves exhaustion costs only the fallback chain, never an allocation event. Refs COD-3222 --- .github/workflows/ci.yml | 2 +- crates/memtrack/testdata/stack_paths.c | 46 +++++ crates/memtrack/tests/shared.rs | 21 ++- crates/memtrack/tests/stack_tests.rs | 240 +++++++++++++++++++++++++ 4 files changed, 304 insertions(+), 5 deletions(-) create mode 100644 crates/memtrack/testdata/stack_paths.c create mode 100644 crates/memtrack/tests/stack_tests.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75ea..3265a919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 00000000..0cee7f43 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 342cbf6b..b2f461b3 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -90,11 +90,14 @@ macro_rules! assert_events_with_marker_for_each_variant { }; } -/// An event's kind and size, without the addresses that differ between runs of -/// the same workload. `Realloc` needs spelling out since its `Debug` includes the -/// old address. +/// An event's kind and size, without the addresses or stack identities that +/// differ between runs of the same workload. pub fn describe_kind(kind: &MemtrackEventKind) -> String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -124,7 +127,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit - + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -238,6 +241,16 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } +/// Track a command with allocation stack capture enabled, returning its events. +pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { + track_command_with_opts( + command, + TrackerOptions::builder() + .stack_copy_size(Some(copy_size)) + .build(), + ) +} + /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 00000000..7a95cd6c --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,240 @@ +#[macro_use] +mod shared; + +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::collections::HashSet; +use std::process::Command; +use tempfile::TempDir; + +const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; + +fn compile_fixture( + name: &str, + temp_dir: &TempDir, +) -> Result> { + shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + name, + temp_dir.path(), + ) +} +fn require_mapping_support() -> bool { + if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { + eprintln!("skipping stack capture test: mapping support is unavailable"); + return false; + } + true +} + +/// The stack identity carried by each allocation and deallocation event that has one. +fn event_hashes(events: &[MemtrackEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } + | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), + _ => None, + }) + .collect() +} + +fn record_hashes(events: &[MemtrackEvent]) -> HashSet { + events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record } => Some(record.hash), + _ => None, + }) + .collect() +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let records: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } => { + Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) + } + _ => None, + }) + .collect(); + + assert!( + records.len() >= 2, + "expected at least two stack records, got {} ({} events)", + records.len(), + events.len() + ); + + let hashes = record_hashes(&events); + assert_eq!( + hashes.len(), + records.len(), + "stack records must be deduplicated by unique hash" + ); + + for (hash, sp, regs, bytes, truncated) in &records { + assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); + assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); + assert!( + !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, + "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", + bytes.len() + ); + assert_eq!( + *truncated, + bytes.len() == COPY_SIZE as usize, + "record {hash:#x} may only be flagged truncated when it filled the budget" + ); + } + + let carried = event_hashes(&events); + assert!( + !carried.is_empty(), + "expected events carrying a captured stack hash" + ); + assert!( + carried.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash must have a matching stack record" + ); + + // The fixture frees every allocation, so both sides must report identities. + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), + "free events must carry their own stack identity" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let carried = event_hashes(&events); + let records = record_hashes(&events); + assert!( + carried.len() > records.len(), + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + carried.len(), + records.len(), + events.len() + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// The largest budget stresses the verifier hardest: the copy loop and its +/// unrolled per-chunk hash both scale with the configured size, so a program +/// that loads at the default can still exceed the instruction limit here. +/// It is also the only budget at which nothing can be budget-limited, because +/// the stack mapping always ends first. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_max", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; + + let truncated: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), + _ => None, + }) + .collect(); + + assert!( + !record_hashes(&events).is_empty(), + "expected stack records at the maximum copy budget" + ); + assert!( + truncated.is_empty(), + "no capture can be budget-limited at the maximum budget: {truncated:#x?}" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// Restores the capture toggle on drop so a failing assertion cannot leak the +/// override into later tests (the suite runs single-threaded). +struct DisableCaptureGuard; + +impl DisableCaptureGuard { + fn set() -> Self { + // SAFETY: tests run with --test-threads 1, so no concurrent env access. + unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; + Self + } +} + +impl Drop for DisableCaptureGuard { + fn drop(&mut self) { + // SAFETY: see `set`. + unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; + let _guard = DisableCaptureGuard::set(); + let (events, thread_handle) = shared::track_binary(&binary)?; + + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), + "disabled capture must still report allocation events" + ); + assert!( + record_hashes(&events).is_empty(), + "disabled capture must emit zero stack records" + ); + assert!( + event_hashes(&events).is_empty(), + "disabled capture must leave stack_hash zero on every event" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} From 706a2b1ac312aca65c4435e1569c45f91db491c0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:44:42 +0200 Subject: [PATCH 05/27] refactor(runner): move ELF artifact pipeline to executor/shared The symbol, unwind-data and debug-info extraction is not perf-specific: it turns a set of mapped ELF modules into the deduplicated keyed artifacts a profile references, whatever discovered the mappings. Memory mode needs the same pipeline, so it moves out of wall_time/profiler/perf into executor/shared/module_artifacts. --- src/executor/helpers/debug_file.rs | 6 ++++++ src/executor/shared/mod.rs | 1 + .../module_artifacts}/debug_info.rs | 0 .../module_artifacts}/elf_helper.rs | 0 .../module_artifacts}/loaded_module.rs | 0 src/executor/shared/module_artifacts/mod.rs | 15 +++++++++++++++ .../module_artifacts}/module_symbols.rs | 0 .../perf => shared/module_artifacts}/naming.rs | 0 .../module_artifacts}/save_artifacts.rs | 2 +- ...facts__debug_info__tests__cpp_debug_info.snap} | 2 +- ...ts__debug_info__tests__golang_debug_info.snap} | 2 +- ...acts__debug_info__tests__ruff_debug_info.snap} | 2 +- ...debug_info__tests__rust_divan_debug_info.snap} | 2 +- ...g_info__tests__the_algorithms_debug_info.snap} | 2 +- ...acts__module_symbols__tests__cpp_symbols.snap} | 2 +- ...s__module_symbols__tests__golang_symbols.snap} | 2 +- ...cts__module_symbols__tests__ruff_symbols.snap} | 2 +- ...odule_symbols__tests__rust_divan_symbols.snap} | 2 +- ...e_symbols__tests__the_algorithms_symbols.snap} | 2 +- ...cts__unwind_data__tests__cpp_unwind_data.snap} | 2 +- ...__unwind_data__tests__golang_unwind_data.snap} | 2 +- ...ts__unwind_data__tests__ruff_unwind_data.snap} | 2 +- ...wind_data__tests__rust_divan_unwind_data.snap} | 2 +- ..._data__tests__the_algorithms_unwind_data.snap} | 2 +- .../module_artifacts}/unwind_data.rs | 0 src/executor/wall_time/profiler/perf/jit_dump.rs | 2 +- src/executor/wall_time/profiler/perf/mod.rs | 8 +------- .../wall_time/profiler/perf/parse_perf_file.rs | 6 +++--- 28 files changed, 43 insertions(+), 27 deletions(-) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/debug_info.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/elf_helper.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/loaded_module.rs (100%) create mode 100644 src/executor/shared/module_artifacts/mod.rs rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/module_symbols.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/naming.rs (100%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/save_artifacts.rs (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap} (99%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap => shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap} (90%) rename src/executor/{wall_time/profiler/perf => shared/module_artifacts}/unwind_data.rs (100%) diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index 0619bed2..e3bbd506 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,6 +12,12 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { + if let Some(dir) = binary_path.parent() { + if let Some(path) = find_debug_file_in(object, binary_path, dir) { + return Some(path); + } + } + ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf406..f278f07c 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 00000000..72c2bf0e --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,15 @@ +//! Extraction of symbols, unwind data and debug info from the ELF modules a +//! profiled process mapped, and their deduplicated on-disk layout. +//! +//! The input is a set of [`loaded_module::LoadedModule`]s, however the mappings +//! were discovered; the output is the keyed `unwind_data`/`symbols.map` files +//! plus the per-pid references that the metadata points at. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 99% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12..9b489942 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,7 +1,7 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d65407..9b917e54 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa..5b6a04ae 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a449..97dc7b43 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a..fd5e1aa0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2..a238cbb2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05..34c79e04 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7b..990e660d 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f9..fe5907dd 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f3..10a77b71 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e280..724f3002 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e14..6c554024 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b03..807e1061 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f9..3956fd7d 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2de..edfd8e55 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca..066c1ad0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad05..344f4080 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb..625407e8 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b5494..1d1033b3 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; From 9a5eef87d4c14f4b73c9a689879ed850dffb9f22 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 19:53:32 +0200 Subject: [PATCH 06/27] feat(runner): add MemtrackMetadata sharing ModuleArtifacts with walltime Memory mode needs the same per-pid module references walltime writes, so the five artifact fields move into a flattened `ModuleArtifacts` shared by both formats; walltime's JSON is unchanged, asserted against output captured from the flat struct. Flattening buffers those fields through serde's `Content`, which unlike serde_json's direct deserializer cannot parse a string JSON key into a pid, so pid-keyed maps get an explicit key-parsing helper. --- crates/runner-shared/src/lib.rs | 1 + crates/runner-shared/src/metadata.rs | 191 ++++++++++++++++-- crates/runner-shared/src/serde_pid_map.rs | 36 ++++ .../shared/module_artifacts/save_artifacts.rs | 25 +-- src/executor/wall_time/profiler/perf/mod.rs | 6 +- src/executor/wall_time/profiler/samply/mod.rs | 6 +- 6 files changed, 229 insertions(+), 36 deletions(-) create mode 100644 crates/runner-shared/src/serde_pid_map.rs diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 61e804de..2cdc7d5a 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,5 +4,6 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; +pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89..654ae298 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -11,35 +11,43 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. #[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, - +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +57,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +109,142 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: 1, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs new file mode 100644 index 00000000..fa6fbb9d --- /dev/null +++ b/crates/runner-shared/src/serde_pid_map.rs @@ -0,0 +1,36 @@ +//! `#[serde(with = ...)]` support for pid-keyed maps. +//! +//! JSON object keys are always strings. serde_json's direct deserializer +//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` +//! field is buffered into serde's internal `Content` first, and that path has no +//! such special case — an `i32` key then fails with `invalid type: string`. So +//! the keys are read as strings and parsed here, which works on both paths. + +use libc::pid_t; +use serde::de::{Deserializer, Error}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::HashMap; + +pub fn serialize(map: &HashMap, serializer: S) -> Result +where + V: Serialize, + S: Serializer, +{ + map.serialize(serializer) +} + +pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} diff --git a/src/executor/shared/module_artifacts/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs index 9b489942..3e8903ed 100644 --- a/src/executor/shared/module_artifacts/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -6,18 +6,17 @@ use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 625407e8..7f31921e 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -300,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ad..5f04ef8c 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), From 263a003e3664094099f0454d56f4cad9b55c3890 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:13:30 +0200 Subject: [PATCH 07/27] feat(memtrack): record mapped modules for offline stack attribution Allocation stacks are raw addresses, so resolving them off-box needs the module geometry perf gets from PERF_RECORD_MMAP2. No single hook provides it: security_mmap_file has the file but runs before the VMA exists, and perf_event_mmap has the addresses but cannot resolve a path. So an LSM program caches the path once per inode and a perf_event_mmap fentry emits inode-keyed address records, joined in userspace while the maps are still live. Path resolution is only reachable from an LSM program at all, and only above 5.11 (bpf_d_path on the sleepable hook) or 6.12 (the bpf_path_d_path kfunc), with the bpf LSM active. MappingSupport probes both, and when neither holds stack capture is turned off rather than shipping stacks nothing can attribute. --- crates/memtrack/src/ebpf/c/attach.h | 5 - crates/memtrack/src/ebpf/c/event.h | 21 +++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/mappings.bpf.h | 166 ++++++++++++++++++ crates/memtrack/src/ebpf/mappings/mod.rs | 7 + crates/memtrack/src/ebpf/mappings/records.rs | 84 +++++++++ crates/memtrack/src/ebpf/mappings/resolve.rs | 77 ++++++++ crates/memtrack/src/ebpf/mappings/support.rs | 103 +++++++++++ crates/memtrack/src/ebpf/memtrack/maps.rs | 52 ++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 56 +++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 +++ crates/memtrack/src/ebpf/mod.rs | 2 + crates/memtrack/src/ebpf/tracker.rs | 69 +++++++- crates/memtrack/src/main.rs | 6 + crates/memtrack/src/session.rs | 3 + .../src/artifacts/memtrack/mappings.rs | 37 ++++ .../src/artifacts/memtrack/mod.rs | 2 + 17 files changed, 700 insertions(+), 14 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h create mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs create mode 100644 crates/memtrack/src/ebpf/mappings/records.rs create mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs create mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index e188c7d5..90cbe436 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,11 +14,6 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 -struct inode_key { - __u64 dev; - __u64 ino; -}; - /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index f4b33c75..eedb0bd3 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -114,6 +114,13 @@ struct event { } data; }; +/* Identifies a mapped file across both the attach watcher and the mapping + * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ +struct inode_key { + uint64_t dev; + uint64_t ino; +}; + /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -121,4 +128,18 @@ struct attach_request { uint64_t ino; }; +/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not + * here: it is resolved once per inode into a BPF map that userspace joins + * against, since every mapping of the same file shares it. */ +struct mapping_record { + uint64_t dev; + uint64_t ino; + uint64_t file_offset; /* offset of the mapping's first byte in the file */ + uint64_t start; + uint64_t end; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + uint32_t pid; + uint32_t _pad; +}; + #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index b405f572..7a068c60 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h new file mode 100644 index 00000000..71b11b39 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/mappings.bpf.h @@ -0,0 +1,166 @@ +#ifndef __MAPPINGS_BPF_H__ +#define __MAPPINGS_BPF_H__ + +#include "event.h" +#include "utils/folio.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* == Mapping recorder == + * + * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked + * process mapped, where, so raw stack addresses can be attributed to modules + * offline. No single hook carries both halves: + * + * security_mmap_file(file, ..) has the file, runs before the VMA exists + * perf_event_mmap(vma) has the addresses, cannot resolve a path + * + * The path therefore lands in a per-inode cache, and the address-bearing hook + * emits inode-keyed records that userspace joins against that cache while this + * BPF object is still loaded. + * + * Path resolution is only reachable from an LSM program: `bpf_d_path()` is + * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist + * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects + * non-LSM program types. Both variants are compiled; userspace autoloads the + * one the running kernel supports and neither when the bpf LSM is inactive. */ + +/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a + * type). Only executable mappings are recorded: unwind data and symbols are + * looked up by text address. */ +#define MEMTRACK_VM_EXEC 0x00000004 + +/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer + * loses whole modules. PATH_MAX keeps that from happening. */ +#define MEMTRACK_MAX_PATH 4096 + +struct inode_path { + __u32 len; /* bytes written by d_path, including the NUL */ + char path[MEMTRACK_MAX_PATH]; +}; + +/* Resolved once per mapped file: a run maps hundreds of distinct files, not + * thousands, and every mapping of the same inode shares the path. */ +BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); + +/* Records are ~64 B and rare (one per executable mapping); the counter below + * reports overflow so the run can fail rather than silently lose a module. */ +BPF_RINGBUF(mappings, 256 * 1024); +BPF_ARRAY_MAP(mapping_dropped, __u64, 1); + +/* An `inode_path` is far larger than the 512 B BPF stack allows, so it is built + * here and copied into the cache from this pointer. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct inode_path); +} path_scratch SEC(".maps"); + +extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; + +static __always_inline void bump_mapping_dropped(void) { + __u32 zero = 0; + __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); + if (drops) { + __sync_fetch_and_add(drops, 1); + } +} + +/* The scratch buffer to resolve `file`'s path into, or NULL when this mapping + * needs no resolution (untracked process, or the inode is already cached). + * `key` is filled in for the matching [`commit_mapping_path`]. */ +static __always_inline struct inode_path* mapping_path_slot(struct file* file, + struct inode_key* key) { + if (!file || !is_tracked(current_tgid())) { + return NULL; + } + + key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + key->ino = BPF_CORE_READ(file, f_inode, i_ino); + if (bpf_map_lookup_elem(&path_by_inode, key)) { + return NULL; + } + + __u32 zero = 0; + return bpf_map_lookup_elem(&path_scratch, &zero); +} + +/* Publish a resolved path. A failed resolution is not cached, so the next + * mapping of the same inode retries instead of losing the module for the run. */ +static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, + int len) { + if (len <= 0) { + return; + } + entry->len = (__u32)len; + bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); +} + +/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ +SEC("lsm/mmap_file") +int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, + bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which + * `mmap_file` has been since 5.11. */ +SEC("lsm.s/mmap_file") +int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* The same hook perf emits MMAP2 from, so the recorded geometry matches what + * the walltime pipeline already consumes: the file offset is in bytes, not + * pages. */ +SEC("fentry/perf_event_mmap") +int BPF_PROG(record_mmap, struct vm_area_struct* vma) { + if (!vma) { + return 0; + } + + __u32 tgid = current_tgid(); + if (!is_tracked(tgid)) { + return 0; + } + + struct file* file = BPF_CORE_READ(vma, vm_file); + if (!file) { + return 0; + } + if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { + return 0; + } + + struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); + if (!rec) { + bump_mapping_dropped(); + return 0; + } + + rec->pid = tgid; + rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + rec->ino = BPF_CORE_READ(file, f_inode, i_ino); + rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; + rec->start = BPF_CORE_READ(vma, vm_start); + rec->end = BPF_CORE_READ(vma, vm_end); + rec->timestamp = bpf_ktime_get_ns(); + bpf_ringbuf_submit(rec, 0); + + return 0; +} + +#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs new file mode 100644 index 00000000..581de916 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/mod.rs @@ -0,0 +1,7 @@ +mod records; +mod resolve; +mod support; + +pub(crate) use records::MappingRecord; +pub(crate) use resolve::resolve_mappings; +pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs new file mode 100644 index 00000000..0d9609ff --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/records.rs @@ -0,0 +1,84 @@ +use crate::ebpf::events::bindings::mapping_record; + +/// One executable file mapping as the BPF recorder saw it. The path is resolved +/// separately, per inode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingRecord { + pub pid: u32, + pub dev: u64, + pub ino: u64, + pub file_offset: u64, + pub start: u64, + pub end: u64, + pub timestamp: u64, +} + +impl MappingRecord { + /// Decode one record from raw ring buffer bytes. + pub fn parse(data: &[u8]) -> Option { + if data.len() < std::mem::size_of::() { + return None; + } + + // SAFETY: the length is checked above, and the layout is the + // bindgen-generated C ABI struct. + let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; + Some(Self { + pid: record.pid, + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + start: record.start, + end: record.end, + timestamp: record.timestamp, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(record: mapping_record) -> Vec { + // SAFETY: reading a plain-data struct as bytes. + unsafe { + std::slice::from_raw_parts( + (&record as *const mapping_record).cast::(), + std::mem::size_of::(), + ) + } + .to_vec() + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let bytes = encode(mapping_record { + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + pid: 7, + _pad: 0, + }); + + assert_eq!( + MappingRecord::parse(&bytes), + Some(MappingRecord { + pid: 7, + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + }) + ); + } + + #[test] + fn truncated_buffer_returns_none() { + assert!(MappingRecord::parse(&[0u8; 8]).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs new file mode 100644 index 00000000..bc2e24ad --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/resolve.rs @@ -0,0 +1,77 @@ +use super::MappingRecord; +use crate::prelude::*; +use runner_shared::artifacts::ProcessMapping; +use std::collections::HashMap; + +/// Join recorded mappings with the per-inode paths resolved in the kernel. +/// +/// A record whose inode has no path is dropped: it was mapped by a process the +/// LSM hook never saw resolve, and without a path there is nothing to read +/// unwind data or symbols from. +pub(crate) fn resolve_mappings( + records: Vec, + paths: &HashMap<(u64, u64), String>, +) -> Vec { + let mut unresolved = 0; + let mappings = records + .into_iter() + .filter_map(|record| { + let Some(path) = paths.get(&(record.dev, record.ino)) else { + unresolved += 1; + return None; + }; + + Some(ProcessMapping { + pid: record.pid as i32, + path: path.clone(), + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + avma_range: record.start..record.end, + timestamp: record.timestamp, + }) + }) + .collect(); + + if unresolved > 0 { + debug!("{unresolved} mapping records had no resolved path and were dropped"); + } + mappings +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(dev: u64, ino: u64) -> MappingRecord { + MappingRecord { + pid: 5, + dev, + ino, + file_offset: 0x1000, + start: 0x4000, + end: 0x8000, + timestamp: 42, + } + } + + #[test] + fn resolves_records_against_the_path_cache() { + let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); + + let mappings = resolve_mappings(vec![record(1, 2)], &paths); + + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, "/lib/libc.so.6"); + assert_eq!(mappings[0].avma_range, 0x4000..0x8000); + assert_eq!(mappings[0].file_offset, 0x1000); + assert_eq!(mappings[0].pid, 5); + } + + /// A module we cannot name is a module we cannot read, so it must not reach + /// the artifact as an empty path. + #[test] + fn drops_records_without_a_resolved_path() { + assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs new file mode 100644 index 00000000..9e2f3a19 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/support.rs @@ -0,0 +1,103 @@ +use crate::kernel::KernelVersion; +use crate::prelude::*; + +/// How the running kernel can resolve a mapped file's path inside BPF. +/// +/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to +/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist +/// that contains no mmap path (`bpf_d_path_allowed()` in +/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces +/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in +/// `fs/bpf_fs_kfuncs.c`). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum MappingSupport { + /// Paths cannot be resolved, so allocation stacks could not be attributed to + /// modules and are not worth capturing. + Unsupported, + /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). + Legacy, + /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). + Kfunc, +} + +impl MappingSupport { + /// What the running kernel and its boot configuration provide. + /// + /// The kernel release is only half the gate: `bpf` must also be in the + /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot + /// be inferred from the version. + pub fn detect() -> Self { + if !bpf_lsm_active() { + info!( + "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ + cannot be resolved" + ); + return Self::Unsupported; + } + + let version = match KernelVersion::current() { + Ok(version) => version, + Err(e) => { + warn!("Failed to read the kernel version, no mapping records: {e:#}"); + return Self::Unsupported; + } + }; + + let support = Self::for_version(version); + match support { + Self::Unsupported => { + info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") + } + Self::Legacy => { + debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") + } + Self::Kfunc => {} + } + support + } + + fn for_version(version: KernelVersion) -> Self { + if version < KernelVersion::new(5, 11) { + return Self::Unsupported; + } + if version < KernelVersion::new(6, 12) { + return Self::Legacy; + } + Self::Kfunc + } +} + +/// Whether `bpf` is one of the LSMs the running kernel initialized. An +/// unreadable file means securityfs is not mounted, in which case no LSM program +/// will attach either. +fn bpf_lsm_active() -> bool { + const PATH: &str = "/sys/kernel/security/lsm"; + + let Ok(active) = std::fs::read_to_string(PATH) else { + debug!("Could not read {PATH} to check whether the bpf LSM is active"); + return false; + }; + active.trim().split(',').any(|lsm| lsm == "bpf") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and + /// 6.12 is the first release carrying `bpf_path_d_path`. + #[test] + fn maps_releases_to_support_levels() { + for (major, minor, expected) in [ + (5, 4, MappingSupport::Unsupported), + (5, 10, MappingSupport::Unsupported), + (5, 11, MappingSupport::Legacy), + (6, 11, MappingSupport::Legacy), + (6, 12, MappingSupport::Kfunc), + (7, 1, MappingSupport::Kfunc), + ] { + let version = KernelVersion::new(major, minor); + assert_eq!(MappingSupport::for_version(version), expected, "{version}"); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 0a60f270..5a4bf355 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,6 +2,7 @@ use super::MemtrackBpf; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; +use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -62,6 +63,39 @@ impl MemtrackBpf { ) } + /// Number of mapping records dropped because their ring buffer was full. + /// A non-zero value means a module may be missing from the trace. + pub fn mapping_dropped_count(&self) -> Result { + read_counter( + with_skel!(self, skel => &skel.maps.mapping_dropped), + "mapping_dropped", + ) + } + + /// The paths the kernel resolved for every mapped file, keyed by + /// `(dev, ino)`. Only readable while the BPF object is alive. + pub fn mapped_paths(&self) -> Result> { + let map = with_skel!(self, skel => &skel.maps.path_by_inode); + + let mut paths = HashMap::new(); + for key in map.keys() { + let Some(value) = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to read a resolved mapping path")? + else { + continue; + }; + + let Some((dev, ino)) = inode_key(&key) else { + continue; + }; + if let Some(path) = inode_path(&value) { + paths.insert((dev, ino), path); + } + } + Ok(paths) + } + pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -118,6 +152,24 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } +/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. +fn inode_key(key: &[u8]) -> Option<(u64, u64)> { + if key.len() < 16 { + return None; + } + Some((le(&key[..8]), le(&key[8..16]))) +} + +/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel +/// wrote `len` bytes including the NUL terminator. +fn inode_path(value: &[u8]) -> Option { + const PATH_OFFSET: usize = 4; + + let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; + let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; + Some(String::from_utf8_lossy(path).into_owned()) +} + /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index df8505e3..72171685 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; +use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -122,28 +123,35 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, + pub(super) mappings: MappingSupport, } impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool, stack_copy_size: Option) -> Result { + pub fn new_with_rmap( + track_rmap: bool, + stack_copy_size: Option, + mappings: MappingSupport, + ) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap, stack_copy_size) + Self::with_variant(variant, track_rmap, stack_copy_size, mappings) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. /// - /// `stack_copy_size` turns on allocation stack capture. + /// `stack_copy_size` turns on allocation stack capture, and `mappings` + /// selects the path-resolving LSM program the running kernel supports. pub fn with_variant( variant: BpfVariant, track_rmap: bool, stack_copy_size: Option, + mappings: MappingSupport, ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { @@ -210,6 +218,26 @@ impl MemtrackBpf { RmapSupport::CoreAndPud => {} } + // The kfunc variant fails to load on kernels without + // `bpf_path_d_path`, and neither LSM program can attach when the + // bpf LSM is inactive; without a path there is nothing to + // resolve records against, so the recorder goes too. + match mappings { + MappingSupport::Unsupported => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + open_skel.progs.record_mmap.set_autoload(false); + open_skel.maps.mappings.set_max_entries(4096)?; + open_skel.maps.path_by_inode.set_max_entries(1)?; + } + MappingSupport::Legacy => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + } + MappingSupport::Kfunc => { + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + } + } + $skel(Box::new( open_skel .load() @@ -231,6 +259,7 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, + mappings, }) } @@ -296,6 +325,27 @@ impl MemtrackBpf { )) } + /// Poll the mapping-record ring buffer into `tx`. Same contract as + /// [`Self::poll_events_with_channel`]. + pub(crate) fn poll_mappings_with_channel( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.mappings, + crate::ebpf::mappings::MappingRecord::parse, + tx, + poll_interval_ms, + )) + } + + /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is + /// worth polling. + pub fn records_mappings(&self) -> bool { + self.mappings != MappingSupport::Unsupported + } + /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 3e00ceb3..350c039e 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,4 +1,5 @@ use super::{MemtrackBpf, RmapSupport}; +use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -64,4 +65,26 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } + + /// Attach the mapping recorder: the LSM hook caching resolved paths and the + /// `perf_event_mmap` fentry emitting the address records. Only the LSM + /// variant the running kernel supports was loaded. + pub fn attach_mapping_recorder(&mut self) -> Result<()> { + let link = match self.mappings { + MappingSupport::Unsupported => return Ok(()), + MappingSupport::Legacy => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) + } + MappingSupport::Kfunc => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) + } + } + .context("Failed to attach the mmap path resolver")?; + self.probes.push(link); + + let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) + .context("Failed to attach the mapping recorder")?; + self.probes.push(link); + Ok(()) + } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index d964ebed..052cb506 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,5 +1,6 @@ mod attach_worker; mod events; +pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -7,6 +8,7 @@ mod spawn; mod stacks; mod tracker; +pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 5536e532..562a6c53 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,4 +1,5 @@ use crate::ebpf::attach_worker::AttachWorker; +use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; use crate::ebpf::stacks::counters::StackCaptureStats; @@ -6,6 +7,7 @@ use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; +use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; @@ -48,6 +50,9 @@ pub struct Tracker { /// The dedup gate spans the whole BPF object, so a second session would /// reference stack records the first one already consumed. stacks_polled: Option, + /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the + /// session is dropped, so the poller's final drain is included. + mapping_rx: Mutex>>, } impl Tracker { @@ -59,9 +64,22 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let copy_size = options.stack_copy_size.map(clamp_copy_size); + let mappings = MappingSupport::detect(); + + // Stacks are raw addresses: without mapped module paths nothing can + // attribute them, so capturing them would only inflate the artifact. + let copy_size = match (options.stack_copy_size.map(clamp_copy_size), mappings) { + (Some(_), MappingSupport::Unsupported) => { + warn!( + "Allocation stack capture needs in-kernel path resolution, which this host \ + cannot provide; disabling it" + ); + None + } + (copy_size, _) => copy_size, + }; Self::build( - MemtrackBpf::new_with_rmap(options.rmap, copy_size)?, + MemtrackBpf::new_with_rmap(options.rmap, copy_size, mappings)?, options.allocators, copy_size.is_some(), ) @@ -72,7 +90,7 @@ impl Tracker { pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; Self::build( - MemtrackBpf::with_variant(variant, track_rmap, None)?, + MemtrackBpf::with_variant(variant, track_rmap, None, MappingSupport::detect())?, true, false, ) @@ -87,6 +105,7 @@ impl Tracker { bpf.attach_tracepoints()?; if allocators { bpf.attach_exec_watcher()?; + bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -101,6 +120,7 @@ impl Tracker { worker: Mutex::new(worker), allocators, stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), + mapping_rx: Mutex::new(None), }) } @@ -137,17 +157,54 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let (poller, stack_poller) = { + let (mapping_tx, mapping_rx) = mpsc::channel(); + let (poller, stack_poller, mapping_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; let stack_poller = capture_stacks .then(|| bpf.poll_stacks(10, tx.clone())) .transpose()?; - (bpf.poll_events_with_channel(10, tx)?, stack_poller) + let mapping_poller = bpf + .records_mappings() + .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + .transpose()?; + ( + bpf.poll_events_with_channel(10, tx)?, + stack_poller, + mapping_poller, + ) }; + *self.mapping_rx.lock() = Some(mapping_rx); resume(pid)?; - Ok(Session::new(child, rx, poller, stack_poller)) + Ok(Session::new( + child, + rx, + poller, + stack_poller, + mapping_poller, + )) + } + + /// The module mappings recorded during the run, joined with the paths the + /// kernel resolved for them. Call after dropping the session so the poller's + /// final drain is included, and before the BPF object is torn down. + pub fn mappings(&self) -> Result { + let Some(rx) = self.mapping_rx.lock().take() else { + return Ok(MemtrackMappings::default()); + }; + + let records: Vec<_> = rx.try_iter().collect(); + let paths = self.bpf.lock().mapped_paths()?; + + let dropped = self.bpf.lock().mapping_dropped_count()?; + if dropped > 0 { + warn!("{dropped} mapping records were dropped; some modules may be unresolved"); + } + + Ok(MemtrackMappings { + mappings: resolve_mappings(records, &paths), + }) } /// Enable allocator-event tracking in the BPF program. Lifetime events diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff19..769d7fca 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,12 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + // Needs the BPF maps, so it has to run before teardown; the session is + // already dropped, so the poller's final drain is in the channel. + let mappings = tracker.mappings().context("Failed to collect mappings")?; + info!("Recorded {} module mappings", mappings.mappings.len()); + mappings.save_with_pid_to(out_dir, root_pid)?; + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 9bed66b7..551db639 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -11,6 +11,7 @@ pub struct Session { events: Option>, _poller: RingBufferPoller, _stack_poller: Option, + _mapping_poller: Option, } impl Session { @@ -19,12 +20,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, + mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, + _mapping_poller: mapping_poller, } } diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs new file mode 100644 index 00000000..b271dfee --- /dev/null +++ b/crates/runner-shared/src/artifacts/memtrack/mappings.rs @@ -0,0 +1,37 @@ +use libc::pid_t; +use serde::{Deserialize, Serialize}; +use std::ops::Range; + +/// The file-backed mappings the tracked process tree loaded, recorded as they +/// happened. Companion to the event stream: allocation stacks are raw +/// addresses, and these are what turns them back into modules. +/// +/// Kept out of the event stream so a consumer that only needs the module set +/// does not have to decode millions of allocation events. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct MemtrackMappings { + pub mappings: Vec, +} + +impl super::super::ArtifactExt for MemtrackMappings {} + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessMapping { + pub pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + pub path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + pub dev: u64, + pub ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + pub file_offset: u64, + pub avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + pub timestamp: u64, +} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 433aab5a..0ba0ced4 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,9 +2,11 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; +mod mappings; mod pipeline; mod writer; +pub use mappings::*; pub use pipeline::*; pub use writer::*; From 4c89273a939686dd4c46d36fa32c11be0661b118 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:28 +0200 Subject: [PATCH 08/27] feat(runner): write memtrack module artifacts and metadata Memory mode now turns the mappings memtrack recorded into the same keyed unwind_data/symbols.map files walltime writes, plus a memtrack.metadata referencing them per pid, so allocation stacks can be unwound off-box. Each mapping's inode is rechecked against the path before its ELF is read: BPF cannot produce a build id, so the recorded (dev, ino) is what proves the file on disk is still the one that was mapped rather than a rebuilt binary whose eh_frame would be bound to the wrong addresses. --- src/executor/memory/executor.rs | 52 +++-- src/executor/memory/mod.rs | 1 + src/executor/memory/module_artifacts.rs | 275 ++++++++++++++++++++++++ 3 files changed, 314 insertions(+), 14 deletions(-) create mode 100644 src/executor/memory/module_artifacts.rs diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a398..d0f44bba 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -163,7 +165,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +175,19 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; - - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); - - Ok(exit_status) + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; + + // Directly write to the profile folder, to avoid having to define another field + marker_result.save_to(&results_folder).unwrap(); + + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +197,20 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + // Without an integration no benchmark ran, which `teardown` reports. + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +248,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +324,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d..9f48a81a 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 00000000..235dc124 --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,275 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use runner_shared::metadata::MemtrackMetadata; +use std::collections::HashMap; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +/// The version of the memtrack metadata format. +const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata { + version: MEMTRACK_METADATA_CURRENT_VERSION, + integration, + artifacts: saved.artifacts, + } + .save_to(profile_folder) +} + +/// Read every mapping artifact in the folder. One is written per tracked root +/// process, so a run with several of them contributes several files. +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + let artifact = MemtrackMappings::decode_from_reader(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?; + mappings.extend(artifact.mappings); + } + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + // The ELF-derived halves are per file, the mounting is per mapping, so + // only the latter is recomputed for a module mapped more than once. + let unwind_data = match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + process_unwind_data.timestamp = Some(mapping.timestamp); + Some((unwind_data, process_unwind_data)) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some((unwind_data, process_unwind_data)) = unwind_data { + loaded_module.unwind_data = Some(unwind_data); + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + /// The whole runner half of the pipeline: a recorded mapping in, keyed + /// unwind/symbol files plus a metadata referencing them out. + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackMappings { + mappings: vec![ProcessMapping { + pid: 1234, + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + avma_range: 0x5555_555a_7000..0x5555_556b_0000, + timestamp: 999, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } +} From 8d152858815f2edcccef57bdb51081a52766f928 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 20:38:57 +0200 Subject: [PATCH 09/27] docs(memtrack): describe the mapping recorder --- crates/memtrack/AGENTS.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d4..819eb584 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,13 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping recorder (`src/ebpf/c/mappings.bpf.h`, `src/ebpf/mappings/`): an LSM program on `mmap_file` resolves each mapped file's path once per inode into `path_by_inode` (`bpf_path_d_path` on kernels >= 6.12, `bpf_d_path` on the sleepable hook from 5.11), and an `fentry/perf_event_mmap` program emits executable-mapping geometry (`dev`/`ino`/`file_offset`/`start`/`end`) on the `mappings` ring buffer. Userspace joins the two into a `MemtrackMappings` artifact, which the runner turns into `unwind_data`/`symbols.map` files and a `memtrack.metadata` so allocation stacks unwind off-box. `MappingSupport::detect()` gates the programs on the kernel release **and** `bpf` being in `/sys/kernel/security/lsm`; with neither available, stack capture is disabled since nothing could attribute the stacks. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `mappings/` (records/resolve/support), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/mappings.bpf.h` + `c/utils/*.h` + `c/allocator.h`. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). From d77d869c0e800f5dbb43b2e6d27e680879354eb8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:37:27 +0200 Subject: [PATCH 10/27] fixup! feat(memtrack): record mapped modules for offline stack attribution --- crates/memtrack/src/ebpf/memtrack/mod.rs | 13 +++-- crates/memtrack/src/ebpf/mod.rs | 1 - crates/memtrack/src/ebpf/stacks/config.rs | 59 +++-------------------- crates/memtrack/src/ebpf/tracker.rs | 31 +++++------- crates/memtrack/tests/shared.rs | 6 +-- crates/memtrack/tests/stack_tests.rs | 47 ++---------------- 6 files changed, 30 insertions(+), 127 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 72171685..8f538822 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -130,7 +130,7 @@ impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. pub fn new_with_rmap( track_rmap: bool, - stack_copy_size: Option, + capture_stacks: bool, mappings: MappingSupport, ) -> Result { let variant = if has_delegated_bpf_token() { @@ -138,19 +138,19 @@ impl MemtrackBpf { } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap, stack_copy_size, mappings) + Self::with_variant(variant, track_rmap, capture_stacks, mappings) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. /// - /// `stack_copy_size` turns on allocation stack capture, and `mappings` + /// `capture_stacks` enables allocation stack capture, and `mappings` /// selects the path-resolving LSM program the running kernel supports. pub fn with_variant( variant: BpfVariant, track_rmap: bool, - stack_copy_size: Option, + capture_stacks: bool, mappings: MappingSupport, ) -> Result { let page_shift = page_shift()?; @@ -181,15 +181,14 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } - if let Some(copy_size) = stack_copy_size { + if capture_stacks { rodata.capture_stacks_enabled = 1; - rodata.stack_copy_size = copy_size; } } // Avoid reserving the stack maps when capture is disabled. A // ring buffer's size must stay a power-of-two page count. - if stack_copy_size.is_none() { + if !capture_stacks { open_skel.maps.stacks.set_max_entries(4096)?; open_skel.maps.stack_traces.set_max_entries(1)?; open_skel.maps.seen_stack_hashes.set_max_entries(1)?; diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 052cb506..743970c8 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -12,6 +12,5 @@ pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs index dd99b8c7..18162b2e 100644 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -1,54 +1,7 @@ -use crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY; -use crate::prelude::*; - -pub const DEFAULT_STACK_COPY_SIZE: u32 = 8192; - -/// The per-allocation stack copy budget, or `None` when capture was explicitly -/// disabled with `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`. Capture is on by default. -pub fn stack_copy_size_from_env() -> Option { - if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref() == Ok("0") { - return None; - } - - let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { - Ok(value) => match value.parse::() { - Ok(size) => size, - Err(error) => { - warn!( - "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" - ); - DEFAULT_STACK_COPY_SIZE - } - }, - Err(_) => DEFAULT_STACK_COPY_SIZE, - }; - - Some(clamp_copy_size(copy_size)) -} - -/// The kernel copies whole chunks, so a budget that is not a multiple of one -/// would hash bytes it never emits. -pub fn clamp_copy_size(copy_size: u32) -> u32 { - const CHUNK: u32 = 512; - (copy_size / CHUNK * CHUNK).clamp(CHUNK, MEMTRACK_MAX_STACK_COPY) -} - -#[cfg(test)] -mod tests { - use super::clamp_copy_size; - - #[test] - fn rounds_down_to_a_whole_chunk() { - assert_eq!(clamp_copy_size(8_700), 8_192); - } - - #[test] - fn clamps_to_low_bound() { - assert_eq!(clamp_copy_size(63), 512); - } - - #[test] - fn clamps_to_high_bound() { - assert_eq!(clamp_copy_size(u32::MAX), 32_256); - } +/// Whether allocation stack capture is enabled by the environment. +pub fn stack_capture_from_env() -> bool { + !matches!( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), + Ok("0") | Ok("false") + ) } diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 562a6c53..13ed9145 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,7 +1,7 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; +use crate::ebpf::stacks::config::stack_capture_from_env; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; @@ -24,10 +24,9 @@ pub struct TrackerOptions { /// Reconstruct per-process RSS from the folio rmap fentry hooks. #[builder(default = false)] pub rmap: bool, - /// Bytes of user stack to copy for each allocation event. `None` leaves - /// stack capture off; values are clamped to the supported range. - #[builder(default = None)] - pub stack_copy_size: Option, + /// Capture allocation call stacks. + #[builder(default = true)] + pub stack_capture: bool, } impl TrackerOptions { @@ -38,7 +37,7 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) - .stack_copy_size(stack_copy_size_from_env()) + .stack_capture(stack_capture_from_env()) .build() } } @@ -68,29 +67,25 @@ impl Tracker { // Stacks are raw addresses: without mapped module paths nothing can // attribute them, so capturing them would only inflate the artifact. - let copy_size = match (options.stack_copy_size.map(clamp_copy_size), mappings) { - (Some(_), MappingSupport::Unsupported) => { - warn!( - "Allocation stack capture needs in-kernel path resolution, which this host \ - cannot provide; disabling it" - ); - None + let capture_stacks = match (options.stack_capture, mappings) { + (true, MappingSupport::Unsupported) => { + warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); + false } - (copy_size, _) => copy_size, + (capture_stacks, _) => capture_stacks, }; Self::build( - MemtrackBpf::new_with_rmap(options.rmap, copy_size, mappings)?, + MemtrackBpf::new_with_rmap(options.rmap, capture_stacks, mappings)?, options.allocators, - copy_size.is_some(), + capture_stacks, ) } - /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; Self::build( - MemtrackBpf::with_variant(variant, track_rmap, None, MappingSupport::detect())?, + MemtrackBpf::with_variant(variant, track_rmap, false, MappingSupport::detect())?, true, false, ) diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index b2f461b3..9bc2e2e9 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -242,12 +242,10 @@ pub fn track_command_with_rmap_maps( } /// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { +pub fn track_command_with_stacks(command: Command) -> TrackResult { track_command_with_opts( command, - TrackerOptions::builder() - .stack_copy_size(Some(copy_size)) - .build(), + TrackerOptions::builder().stack_capture(true).build(), ) } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 7a95cd6c..ae3e30f2 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -6,7 +6,7 @@ use std::collections::HashSet; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; +const COPY_SIZE: u32 = 8192; fn compile_fixture( name: &str, @@ -59,8 +59,7 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box = events .iter() @@ -133,8 +132,7 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Result<(), Box Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_max", &temp_dir)?; - let (events, thread_handle) = - shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; - - let truncated: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), - _ => None, - }) - .collect(); - - assert!( - !record_hashes(&events).is_empty(), - "expected stack records at the maximum copy budget" - ); - assert!( - truncated.is_empty(), - "no capture can be budget-limited at the maximum budget: {truncated:#x?}" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) -} - /// Restores the capture toggle on drop so a failing assertion cannot leak the /// override into later tests (the suite runs single-threaded). struct DisableCaptureGuard; From 73d1e806f3d9e18d41243d54323cff9b743ab050 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:40:52 +0200 Subject: [PATCH 11/27] test(memtrack): add nested allocation fixtures --- crates/memtrack/testdata/nested_doubling.c | 41 +++++++++++++++++ .../testdata/nested_doubling_shared_free.c | 44 +++++++++++++++++++ crates/memtrack/tests/c_tests.rs | 10 +++++ .../snapshots/c_tests__nested_doubling.snap | 12 +++++ .../c_tests__nested_doubling_shared_free.snap | 12 +++++ 5 files changed, 119 insertions(+) create mode 100644 crates/memtrack/testdata/nested_doubling.c create mode 100644 crates/memtrack/testdata/nested_doubling_shared_free.c create mode 100644 crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap create mode 100644 crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap diff --git a/crates/memtrack/testdata/nested_doubling.c b/crates/memtrack/testdata/nested_doubling.c new file mode 100644 index 00000000..e3a588e3 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling.c @@ -0,0 +1,41 @@ +#include +#include + +/* + * Each level allocates twice as much as its caller, then frees on the way back + * up, so the free order is the reverse of the allocation order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(1024) <-- free(2048) <-- free(4096) + * + * Every malloc and every free sits at a distinct call depth, so the six events + * also carry six distinct allocation stacks. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); +} + +__attribute__((noinline)) static void level2(void) { + void* p = malloc(2048); + escaped_pointer = p; + level3(); + free(p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(); + free(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/nested_doubling_shared_free.c b/crates/memtrack/testdata/nested_doubling_shared_free.c new file mode 100644 index 00000000..66c53fd9 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling_shared_free.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Same doubling allocation chain as nested_doubling.c, but ownership is handed + * down and the innermost level frees all three buffers in reverse order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(4096) + * free(2048) + * free(1024) + * + * The three mallocs come from three different call depths while all three frees + * share one, so a deallocation event must be attributed to the free site rather + * than to wherever its allocation happened. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void* outer, void* middle) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); + free(middle); + free(outer); +} + +__attribute__((noinline)) static void level2(void* outer) { + void* p = malloc(2048); + escaped_pointer = p; + level3(outer, p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 2d33dccf..62b13028 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -51,6 +51,14 @@ const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ name: "posix_memalign_einval", source: include_str!("../testdata/posix_memalign_einval.c"), }, + AllocationTestCase { + name: "nested_doubling", + source: include_str!("../testdata/nested_doubling.c"), + }, + AllocationTestCase { + name: "nested_doubling_shared_free", + source: include_str!("../testdata/nested_doubling_shared_free.c"), + }, ]; #[test_with::env(GITHUB_ACTIONS)] @@ -65,6 +73,8 @@ const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ #[case(&ALLOCATION_TEST_CASES[7])] #[case(&ALLOCATION_TEST_CASES[8])] #[case(&ALLOCATION_TEST_CASES[9])] +#[case(&ALLOCATION_TEST_CASES[10])] +#[case(&ALLOCATION_TEST_CASES[11])] #[test_log::test] fn test_allocation_tracking( #[case] test_case: &AllocationTestCase, diff --git a/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap new file mode 100644 index 00000000..48a216de --- /dev/null +++ b/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/c_tests.rs +expression: formatted_events +--- +[ + "Malloc { size: 1024 }", + "Malloc { size: 2048 }", + "Malloc { size: 4096 }", + "Free", + "Free", + "Free", +] diff --git a/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap new file mode 100644 index 00000000..48a216de --- /dev/null +++ b/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/c_tests.rs +expression: formatted_events +--- +[ + "Malloc { size: 1024 }", + "Malloc { size: 2048 }", + "Malloc { size: 4096 }", + "Free", + "Free", + "Free", +] From ccd824955739ee93f076e54737c71c0c809ad0b6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 12:41:18 +0200 Subject: [PATCH 12/27] test(memtrack): assert nested stack identities --- crates/memtrack/tests/stack_tests.rs | 84 ++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index ae3e30f2..28950dd7 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -197,3 +197,87 @@ fn explicit_disable_suppresses_stack_capture() -> Result<(), Box Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + for (name, source) in [ + ( + "nested_doubling", + include_str!("../testdata/nested_doubling.c"), + ), + ( + "nested_doubling_shared_free", + include_str!("../testdata/nested_doubling_shared_free.c"), + ), + ] { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; + + let allocations: Vec<(u64, u64, u64)> = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { + Some((size, e.addr, stack_hash)) + } + _ => None, + }) + .collect(); + + assert_eq!( + allocations + .iter() + .map(|(size, ..)| *size) + .collect::>(), + vec![1024, 2048, 4096], + "[{name}] each level must allocate twice its caller, outermost first" + ); + + // Both probes of one free report the same address, so dedup by address + // to recover the order the fixture released its buffers in. + let mut released: Vec = Vec::new(); + let mut free_hashes: Vec = Vec::new(); + for event in &events { + let MemtrackEventKind::Free { stack_hash } = event.kind else { + continue; + }; + if released.last() == Some(&event.addr) { + continue; + } + released.push(event.addr); + free_hashes.push(stack_hash); + } + + let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); + let expected: Vec = allocated.iter().rev().copied().collect(); + assert_eq!( + released, expected, + "[{name}] buffers must be freed in the reverse of their allocation order" + ); + + let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); + assert_eq!( + alloc_hashes.len(), + allocations.len(), + "[{name}] each allocation depth must carry its own stack identity" + ); + assert!( + !alloc_hashes.contains(&0) && !free_hashes.contains(&0), + "[{name}] every allocation and free must carry a captured stack" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + } + Ok(()) +} From bc0806db522f9979de35e71528a902e53c21e823 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 13:17:34 +0200 Subject: [PATCH 13/27] fixup! feat(memtrack): capture allocation stacks in eBPF Hardcode the stack copy length and remove redundant comments and fixed-size checks. --- crates/memtrack/src/ebpf/c/stack_capture.bpf.h | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h index 28f7fcf8..ca9eda35 100644 --- a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -18,7 +18,6 @@ */ const volatile __u8 capture_stacks_enabled = 0; -const volatile __u32 stack_copy_size = 8192; #define STACK_TRACE_MAX_DEPTH 127 /* Copy granularity: a recovered length is exact only to within one chunk. */ @@ -109,7 +108,6 @@ static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_re #error "stack capture needs a DWARF register mapping for this architecture" #endif -/* Returns the stack identity, or 0 when nothing could be copied. */ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { __u32 zero = 0; struct stack_scratch_buf* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); @@ -118,14 +116,7 @@ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct tas } __u64 sp = PT_REGS_SP(ctx); - __u32 want = stack_copy_size; - if (want > MEMTRACK_MAX_STACK_COPY) { - want = MEMTRACK_MAX_STACK_COPY; - } - want &= ~(__u32)(STACK_COPY_CHUNK - 1); - if (want < STACK_COPY_CHUNK) { - want = STACK_COPY_CHUNK; - } + const __u32 want = 8192; /* bpf_probe_read_user() is all-or-nothing and the readable region ends at * the top of the stack mapping, which is not knowable up front, so the copy @@ -210,8 +201,6 @@ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct tas return hash; } -/* Copy and hash the caller's stack, emitting a record on first sight of the - * resulting identity. Returns 0 when capture is off or nothing was copied. */ static __always_inline __u64 capture_stack(struct pt_regs* ctx) { if (!capture_stacks_enabled || !is_enabled()) { return 0; @@ -243,7 +232,6 @@ static __always_inline __u64 capture_stack(struct pt_regs* ctx) { return hash; } -/* Hand an identity to the matching uretprobe. */ static __always_inline void stash_stack_hash(__u64 hash) { if (hash == 0) { return; @@ -253,9 +241,7 @@ static __always_inline void stash_stack_hash(__u64 hash) { bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); } -/* The identity stashed by the matching entry probe, or 0 when capture is off or - * the entry probe bailed out. The slot is per-thread but shared by allocators, - * so every return path must clear it. */ +/* Every return path must clear the per-thread slot. */ static __always_inline __u64 take_stack_hash(void) { if (!capture_stacks_enabled) { return 0; From a89944bc89cf7987ed999428f0793c2b75963c42 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 13:17:42 +0200 Subject: [PATCH 14/27] fixup! feat(memtrack): record mapped modules for offline stack attribution Trim mapping comments while keeping the path-resolution and teardown invariants explicit. --- crates/memtrack/src/ebpf/c/mappings.bpf.h | 13 ++++--------- crates/memtrack/src/ebpf/mappings/records.rs | 1 - crates/memtrack/src/ebpf/mappings/resolve.rs | 6 +----- crates/memtrack/src/main.rs | 3 +-- 4 files changed, 6 insertions(+), 17 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h index 71b11b39..5b5635a1 100644 --- a/crates/memtrack/src/ebpf/c/mappings.bpf.h +++ b/crates/memtrack/src/ebpf/c/mappings.bpf.h @@ -39,17 +39,14 @@ struct inode_path { char path[MEMTRACK_MAX_PATH]; }; -/* Resolved once per mapped file: a run maps hundreds of distinct files, not - * thousands, and every mapping of the same inode shares the path. */ +/* Every mapping of an inode shares its cached path. */ BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); -/* Records are ~64 B and rare (one per executable mapping); the counter below - * reports overflow so the run can fail rather than silently lose a module. */ +/* A dropped record may leave a module unresolved. */ BPF_RINGBUF(mappings, 256 * 1024); BPF_ARRAY_MAP(mapping_dropped, __u64, 1); -/* An `inode_path` is far larger than the 512 B BPF stack allows, so it is built - * here and copied into the cache from this pointer. */ +/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); @@ -67,9 +64,7 @@ static __always_inline void bump_mapping_dropped(void) { } } -/* The scratch buffer to resolve `file`'s path into, or NULL when this mapping - * needs no resolution (untracked process, or the inode is already cached). - * `key` is filled in for the matching [`commit_mapping_path`]. */ +/* Return a scratch slot when this inode has no cached path. */ static __always_inline struct inode_path* mapping_path_slot(struct file* file, struct inode_key* key) { if (!file || !is_tracked(current_tgid())) { diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs index 0d9609ff..4c660031 100644 --- a/crates/memtrack/src/ebpf/mappings/records.rs +++ b/crates/memtrack/src/ebpf/mappings/records.rs @@ -14,7 +14,6 @@ pub struct MappingRecord { } impl MappingRecord { - /// Decode one record from raw ring buffer bytes. pub fn parse(data: &[u8]) -> Option { if data.len() < std::mem::size_of::() { return None; diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs index bc2e24ad..9c20ba7f 100644 --- a/crates/memtrack/src/ebpf/mappings/resolve.rs +++ b/crates/memtrack/src/ebpf/mappings/resolve.rs @@ -3,11 +3,7 @@ use crate::prelude::*; use runner_shared::artifacts::ProcessMapping; use std::collections::HashMap; -/// Join recorded mappings with the per-inode paths resolved in the kernel. -/// -/// A record whose inode has no path is dropped: it was mapped by a process the -/// LSM hook never saw resolve, and without a path there is nothing to read -/// unwind data or symbols from. +/// Records without a path are dropped because their unwind data and symbols cannot be read. pub(crate) fn resolve_mappings( records: Vec, paths: &HashMap<(u64, u64), String>, diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 769d7fca..30b9567e 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,8 +159,7 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; - // Needs the BPF maps, so it has to run before teardown; the session is - // already dropped, so the poller's final drain is in the channel. + // Collect after the poller drains and before the BPF maps are torn down. let mappings = tracker.mappings().context("Failed to collect mappings")?; info!("Recorded {} module mappings", mappings.mappings.len()); mappings.save_with_pid_to(out_dir, root_pid)?; From 8b98b38b3dbccad0a7410014de07bd95ec00e219 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 13:17:56 +0200 Subject: [PATCH 15/27] fixup! feat(memtrack): add userspace stack-capture module Remove the redundant environment-helper comment. --- crates/memtrack/src/ebpf/stacks/config.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs index 18162b2e..396817ad 100644 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -1,4 +1,3 @@ -/// Whether allocation stack capture is enabled by the environment. pub fn stack_capture_from_env() -> bool { !matches!( std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), From 80efbcf8b16eb077920317ead08f38159a37630d Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 13:18:10 +0200 Subject: [PATCH 16/27] fixup! feat(runner): write memtrack module artifacts and metadata Remove comments that only narrate implementation details and test setup. --- src/executor/memory/executor.rs | 2 -- src/executor/memory/module_artifacts.rs | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index d0f44bba..b8a294d8 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -183,7 +183,6 @@ impl Executor for MemoryExecutor { Self::handle_fifo(runner_fifo, ipc, &mut child).await?; *integration.borrow_mut() = fifo_data.integration; - // Directly write to the profile folder, to avoid having to define another field marker_result.save_to(&results_folder).unwrap(); Ok(exit_status) @@ -197,7 +196,6 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } - // Without an integration no benchmark ran, which `teardown` reports. if let Some(integration) = integration.borrow_mut().take() { let results_folder = execution_context.profile_folder.join("results"); if let Err(e) = save_module_artifacts( diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index 235dc124..f8dcd0e9 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -9,7 +9,6 @@ use std::collections::HashMap; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -/// The version of the memtrack metadata format. const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; /// Turn the mappings memtrack recorded into the artifacts an offline unwinder @@ -216,8 +215,6 @@ mod tests { )); } - /// The whole runner half of the pipeline: a recorded mapping in, keyed - /// unwind/symbol files plus a metadata referencing them out. #[test] fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; From a42e3f35753906e14e9a2c6896d61bf066b58ee9 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 20:42:02 +0200 Subject: [PATCH 17/27] refactor(memtrack): capture module mappings with perf Replace the BPF LSM path cache and mapping ring with inherited per-CPU PERF_RECORD_MMAP2 collectors. Store executable mappings as a terminal suffix in the main memtrack stream so existing timeline consumers remain compatible, then extract and order them in the runner before generating module artifacts. --- crates/memtrack/src/ebpf/c/event.h | 18 +- crates/memtrack/src/ebpf/c/main.bpf.c | 1 - crates/memtrack/src/ebpf/c/mappings.bpf.h | 161 ------ crates/memtrack/src/ebpf/mappings/mod.rs | 7 - crates/memtrack/src/ebpf/mappings/records.rs | 83 --- crates/memtrack/src/ebpf/mappings/resolve.rs | 73 --- crates/memtrack/src/ebpf/mappings/support.rs | 103 ---- crates/memtrack/src/ebpf/memtrack/maps.rs | 52 -- crates/memtrack/src/ebpf/memtrack/mod.rs | 57 +- crates/memtrack/src/ebpf/memtrack/tracking.rs | 23 - crates/memtrack/src/ebpf/mod.rs | 2 - crates/memtrack/src/ebpf/tracker.rs | 119 ++-- crates/memtrack/src/lib.rs | 2 + crates/memtrack/src/main.rs | 5 - crates/memtrack/src/perf_mappings.rs | 511 ++++++++++++++++++ crates/memtrack/src/session.rs | 13 +- crates/memtrack/tests/stack_tests.rs | 16 - .../src/artifacts/memtrack/mappings.rs | 13 - .../src/artifacts/memtrack/mod.rs | 29 + src/executor/memory/module_artifacts.rs | 227 +++++++- 20 files changed, 820 insertions(+), 695 deletions(-) delete mode 100644 crates/memtrack/src/ebpf/c/mappings.bpf.h delete mode 100644 crates/memtrack/src/ebpf/mappings/mod.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/records.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/resolve.rs delete mode 100644 crates/memtrack/src/ebpf/mappings/support.rs create mode 100644 crates/memtrack/src/perf_mappings.rs diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index eedb0bd3..e413007b 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -114,8 +114,8 @@ struct event { } data; }; -/* Identifies a mapped file across both the attach watcher and the mapping - * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ +/* Identifies a mapped file for the exec-mapping watcher. `dev` uses the + * kernel's s_dev encoding: (major << 20) | minor. */ struct inode_key { uint64_t dev; uint64_t ino; @@ -128,18 +128,4 @@ struct attach_request { uint64_t ino; }; -/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not - * here: it is resolved once per inode into a BPF map that userspace joins - * against, since every mapping of the same file shares it. */ -struct mapping_record { - uint64_t dev; - uint64_t ino; - uint64_t file_offset; /* offset of the mapping's first byte in the file */ - uint64_t start; - uint64_t end; - uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ - uint32_t pid; - uint32_t _pad; -}; - #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 7a068c60..b405f572 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,7 +8,6 @@ #include "allocator.h" #include "attach.h" #include "event.h" -#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h deleted file mode 100644 index 5b5635a1..00000000 --- a/crates/memtrack/src/ebpf/c/mappings.bpf.h +++ /dev/null @@ -1,161 +0,0 @@ -#ifndef __MAPPINGS_BPF_H__ -#define __MAPPINGS_BPF_H__ - -#include "event.h" -#include "utils/folio.h" -#include "utils/map_helpers.h" -#include "utils/process_tracking.h" - -/* == Mapping recorder == - * - * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked - * process mapped, where, so raw stack addresses can be attributed to modules - * offline. No single hook carries both halves: - * - * security_mmap_file(file, ..) has the file, runs before the VMA exists - * perf_event_mmap(vma) has the addresses, cannot resolve a path - * - * The path therefore lands in a per-inode cache, and the address-bearing hook - * emits inode-keyed records that userspace joins against that cache while this - * BPF object is still loaded. - * - * Path resolution is only reachable from an LSM program: `bpf_d_path()` is - * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist - * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects - * non-LSM program types. Both variants are compiled; userspace autoloads the - * one the running kernel supports and neither when the bpf LSM is inactive. */ - -/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a - * type). Only executable mappings are recorded: unwind data and symbols are - * looked up by text address. */ -#define MEMTRACK_VM_EXEC 0x00000004 - -/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer - * loses whole modules. PATH_MAX keeps that from happening. */ -#define MEMTRACK_MAX_PATH 4096 - -struct inode_path { - __u32 len; /* bytes written by d_path, including the NUL */ - char path[MEMTRACK_MAX_PATH]; -}; - -/* Every mapping of an inode shares its cached path. */ -BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); - -/* A dropped record may leave a module unresolved. */ -BPF_RINGBUF(mappings, 256 * 1024); -BPF_ARRAY_MAP(mapping_dropped, __u64, 1); - -/* The path does not fit on the BPF stack; build it in this per-CPU scratch map. */ -struct { - __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); - __uint(max_entries, 1); - __type(key, __u32); - __type(value, struct inode_path); -} path_scratch SEC(".maps"); - -extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; - -static __always_inline void bump_mapping_dropped(void) { - __u32 zero = 0; - __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); - if (drops) { - __sync_fetch_and_add(drops, 1); - } -} - -/* Return a scratch slot when this inode has no cached path. */ -static __always_inline struct inode_path* mapping_path_slot(struct file* file, - struct inode_key* key) { - if (!file || !is_tracked(current_tgid())) { - return NULL; - } - - key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - key->ino = BPF_CORE_READ(file, f_inode, i_ino); - if (bpf_map_lookup_elem(&path_by_inode, key)) { - return NULL; - } - - __u32 zero = 0; - return bpf_map_lookup_elem(&path_scratch, &zero); -} - -/* Publish a resolved path. A failed resolution is not cached, so the next - * mapping of the same inode retries instead of losing the module for the run. */ -static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, - int len) { - if (len <= 0) { - return; - } - entry->len = (__u32)len; - bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); -} - -/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ -SEC("lsm/mmap_file") -int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, - bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which - * `mmap_file` has been since 5.11. */ -SEC("lsm.s/mmap_file") -int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, - unsigned long flags) { - struct inode_key key = {}; - struct inode_path* entry = mapping_path_slot(file, &key); - if (entry) { - commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); - } - return 0; -} - -/* The same hook perf emits MMAP2 from, so the recorded geometry matches what - * the walltime pipeline already consumes: the file offset is in bytes, not - * pages. */ -SEC("fentry/perf_event_mmap") -int BPF_PROG(record_mmap, struct vm_area_struct* vma) { - if (!vma) { - return 0; - } - - __u32 tgid = current_tgid(); - if (!is_tracked(tgid)) { - return 0; - } - - struct file* file = BPF_CORE_READ(vma, vm_file); - if (!file) { - return 0; - } - if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { - return 0; - } - - struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); - if (!rec) { - bump_mapping_dropped(); - return 0; - } - - rec->pid = tgid; - rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); - rec->ino = BPF_CORE_READ(file, f_inode, i_ino); - rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; - rec->start = BPF_CORE_READ(vma, vm_start); - rec->end = BPF_CORE_READ(vma, vm_end); - rec->timestamp = bpf_ktime_get_ns(); - bpf_ringbuf_submit(rec, 0); - - return 0; -} - -#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs deleted file mode 100644 index 581de916..00000000 --- a/crates/memtrack/src/ebpf/mappings/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -mod records; -mod resolve; -mod support; - -pub(crate) use records::MappingRecord; -pub(crate) use resolve::resolve_mappings; -pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs deleted file mode 100644 index 4c660031..00000000 --- a/crates/memtrack/src/ebpf/mappings/records.rs +++ /dev/null @@ -1,83 +0,0 @@ -use crate::ebpf::events::bindings::mapping_record; - -/// One executable file mapping as the BPF recorder saw it. The path is resolved -/// separately, per inode. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct MappingRecord { - pub pid: u32, - pub dev: u64, - pub ino: u64, - pub file_offset: u64, - pub start: u64, - pub end: u64, - pub timestamp: u64, -} - -impl MappingRecord { - pub fn parse(data: &[u8]) -> Option { - if data.len() < std::mem::size_of::() { - return None; - } - - // SAFETY: the length is checked above, and the layout is the - // bindgen-generated C ABI struct. - let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; - Some(Self { - pid: record.pid, - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - start: record.start, - end: record.end, - timestamp: record.timestamp, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn encode(record: mapping_record) -> Vec { - // SAFETY: reading a plain-data struct as bytes. - unsafe { - std::slice::from_raw_parts( - (&record as *const mapping_record).cast::(), - std::mem::size_of::(), - ) - } - .to_vec() - } - - #[test] - fn well_formed_record_round_trips_every_field() { - let bytes = encode(mapping_record { - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - pid: 7, - _pad: 0, - }); - - assert_eq!( - MappingRecord::parse(&bytes), - Some(MappingRecord { - pid: 7, - dev: 0x1_0002, - ino: 4242, - file_offset: 0x2000, - start: 0x5555_5555_0000, - end: 0x5555_5556_0000, - timestamp: 987_654_321, - }) - ); - } - - #[test] - fn truncated_buffer_returns_none() { - assert!(MappingRecord::parse(&[0u8; 8]).is_none()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs deleted file mode 100644 index 9c20ba7f..00000000 --- a/crates/memtrack/src/ebpf/mappings/resolve.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::MappingRecord; -use crate::prelude::*; -use runner_shared::artifacts::ProcessMapping; -use std::collections::HashMap; - -/// Records without a path are dropped because their unwind data and symbols cannot be read. -pub(crate) fn resolve_mappings( - records: Vec, - paths: &HashMap<(u64, u64), String>, -) -> Vec { - let mut unresolved = 0; - let mappings = records - .into_iter() - .filter_map(|record| { - let Some(path) = paths.get(&(record.dev, record.ino)) else { - unresolved += 1; - return None; - }; - - Some(ProcessMapping { - pid: record.pid as i32, - path: path.clone(), - dev: record.dev, - ino: record.ino, - file_offset: record.file_offset, - avma_range: record.start..record.end, - timestamp: record.timestamp, - }) - }) - .collect(); - - if unresolved > 0 { - debug!("{unresolved} mapping records had no resolved path and were dropped"); - } - mappings -} - -#[cfg(test)] -mod tests { - use super::*; - - fn record(dev: u64, ino: u64) -> MappingRecord { - MappingRecord { - pid: 5, - dev, - ino, - file_offset: 0x1000, - start: 0x4000, - end: 0x8000, - timestamp: 42, - } - } - - #[test] - fn resolves_records_against_the_path_cache() { - let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); - - let mappings = resolve_mappings(vec![record(1, 2)], &paths); - - assert_eq!(mappings.len(), 1); - assert_eq!(mappings[0].path, "/lib/libc.so.6"); - assert_eq!(mappings[0].avma_range, 0x4000..0x8000); - assert_eq!(mappings[0].file_offset, 0x1000); - assert_eq!(mappings[0].pid, 5); - } - - /// A module we cannot name is a module we cannot read, so it must not reach - /// the artifact as an empty path. - #[test] - fn drops_records_without_a_resolved_path() { - assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); - } -} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs deleted file mode 100644 index 9e2f3a19..00000000 --- a/crates/memtrack/src/ebpf/mappings/support.rs +++ /dev/null @@ -1,103 +0,0 @@ -use crate::kernel::KernelVersion; -use crate::prelude::*; - -/// How the running kernel can resolve a mapped file's path inside BPF. -/// -/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to -/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist -/// that contains no mmap path (`bpf_d_path_allowed()` in -/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces -/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in -/// `fs/bpf_fs_kfuncs.c`). -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum MappingSupport { - /// Paths cannot be resolved, so allocation stacks could not be attributed to - /// modules and are not worth capturing. - Unsupported, - /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). - Legacy, - /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). - Kfunc, -} - -impl MappingSupport { - /// What the running kernel and its boot configuration provide. - /// - /// The kernel release is only half the gate: `bpf` must also be in the - /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot - /// be inferred from the version. - pub fn detect() -> Self { - if !bpf_lsm_active() { - info!( - "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ - cannot be resolved" - ); - return Self::Unsupported; - } - - let version = match KernelVersion::current() { - Ok(version) => version, - Err(e) => { - warn!("Failed to read the kernel version, no mapping records: {e:#}"); - return Self::Unsupported; - } - }; - - let support = Self::for_version(version); - match support { - Self::Unsupported => { - info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") - } - Self::Legacy => { - debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") - } - Self::Kfunc => {} - } - support - } - - fn for_version(version: KernelVersion) -> Self { - if version < KernelVersion::new(5, 11) { - return Self::Unsupported; - } - if version < KernelVersion::new(6, 12) { - return Self::Legacy; - } - Self::Kfunc - } -} - -/// Whether `bpf` is one of the LSMs the running kernel initialized. An -/// unreadable file means securityfs is not mounted, in which case no LSM program -/// will attach either. -fn bpf_lsm_active() -> bool { - const PATH: &str = "/sys/kernel/security/lsm"; - - let Ok(active) = std::fs::read_to_string(PATH) else { - debug!("Could not read {PATH} to check whether the bpf LSM is active"); - return false; - }; - active.trim().split(',').any(|lsm| lsm == "bpf") -} - -#[cfg(test)] -mod tests { - use super::*; - - /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and - /// 6.12 is the first release carrying `bpf_path_d_path`. - #[test] - fn maps_releases_to_support_levels() { - for (major, minor, expected) in [ - (5, 4, MappingSupport::Unsupported), - (5, 10, MappingSupport::Unsupported), - (5, 11, MappingSupport::Legacy), - (6, 11, MappingSupport::Legacy), - (6, 12, MappingSupport::Kfunc), - (7, 1, MappingSupport::Kfunc), - ] { - let version = KernelVersion::new(major, minor); - assert_eq!(MappingSupport::for_version(version), expected, "{version}"); - } - } -} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 5a4bf355..0a60f270 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -2,7 +2,6 @@ use super::MemtrackBpf; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; -use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -63,39 +62,6 @@ impl MemtrackBpf { ) } - /// Number of mapping records dropped because their ring buffer was full. - /// A non-zero value means a module may be missing from the trace. - pub fn mapping_dropped_count(&self) -> Result { - read_counter( - with_skel!(self, skel => &skel.maps.mapping_dropped), - "mapping_dropped", - ) - } - - /// The paths the kernel resolved for every mapped file, keyed by - /// `(dev, ino)`. Only readable while the BPF object is alive. - pub fn mapped_paths(&self) -> Result> { - let map = with_skel!(self, skel => &skel.maps.path_by_inode); - - let mut paths = HashMap::new(); - for key in map.keys() { - let Some(value) = map - .lookup(&key, libbpf_rs::MapFlags::ANY) - .context("Failed to read a resolved mapping path")? - else { - continue; - }; - - let Some((dev, ino)) = inode_key(&key) else { - continue; - }; - if let Some(path) = inode_path(&value) { - paths.insert((dev, ino), path); - } - } - Ok(paths) - } - pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -152,24 +118,6 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } -/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. -fn inode_key(key: &[u8]) -> Option<(u64, u64)> { - if key.len() < 16 { - return None; - } - Some((le(&key[..8]), le(&key[8..16]))) -} - -/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel -/// wrote `len` bytes including the NUL terminator. -fn inode_path(value: &[u8]) -> Option { - const PATH_OFFSET: usize = 4; - - let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; - let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; - Some(String::from_utf8_lossy(path).into_owned()) -} - /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 8f538822..0644b176 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,7 +6,6 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; -use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -123,35 +122,27 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, - pub(super) mappings: MappingSupport, } impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap( - track_rmap: bool, - capture_stacks: bool, - mappings: MappingSupport, - ) -> Result { + pub fn new_with_rmap(track_rmap: bool, capture_stacks: bool) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap, capture_stacks, mappings) + Self::with_variant(variant, track_rmap, capture_stacks) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. - /// - /// `capture_stacks` enables allocation stack capture, and `mappings` - /// selects the path-resolving LSM program the running kernel supports. + /// `capture_stacks` enables allocation stack capture. pub fn with_variant( variant: BpfVariant, track_rmap: bool, capture_stacks: bool, - mappings: MappingSupport, ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { @@ -217,26 +208,6 @@ impl MemtrackBpf { RmapSupport::CoreAndPud => {} } - // The kfunc variant fails to load on kernels without - // `bpf_path_d_path`, and neither LSM program can attach when the - // bpf LSM is inactive; without a path there is nothing to - // resolve records against, so the recorder goes too. - match mappings { - MappingSupport::Unsupported => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - open_skel.progs.record_mmap.set_autoload(false); - open_skel.maps.mappings.set_max_entries(4096)?; - open_skel.maps.path_by_inode.set_max_entries(1)?; - } - MappingSupport::Legacy => { - open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); - } - MappingSupport::Kfunc => { - open_skel.progs.cache_mmap_path_legacy.set_autoload(false); - } - } - $skel(Box::new( open_skel .load() @@ -258,7 +229,6 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, - mappings, }) } @@ -324,27 +294,6 @@ impl MemtrackBpf { )) } - /// Poll the mapping-record ring buffer into `tx`. Same contract as - /// [`Self::poll_events_with_channel`]. - pub(crate) fn poll_mappings_with_channel( - &self, - poll_interval_ms: u64, - tx: std::sync::mpsc::Sender, - ) -> Result { - with_skel!(self, skel => RingBufferPoller::new( - &skel.maps.mappings, - crate::ebpf::mappings::MappingRecord::parse, - tx, - poll_interval_ms, - )) - } - - /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is - /// worth polling. - pub fn records_mappings(&self) -> bool { - self.mappings != MappingSupport::Unsupported - } - /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 350c039e..3e00ceb3 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,5 +1,4 @@ use super::{MemtrackBpf, RmapSupport}; -use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -65,26 +64,4 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } - - /// Attach the mapping recorder: the LSM hook caching resolved paths and the - /// `perf_event_mmap` fentry emitting the address records. Only the LSM - /// variant the running kernel supports was loaded. - pub fn attach_mapping_recorder(&mut self) -> Result<()> { - let link = match self.mappings { - MappingSupport::Unsupported => return Ok(()), - MappingSupport::Legacy => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) - } - MappingSupport::Kfunc => { - with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) - } - } - .context("Failed to attach the mmap path resolver")?; - self.probes.push(link); - - let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) - .context("Failed to attach the mapping recorder")?; - self.probes.push(link); - Ok(()) - } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 743970c8..5b482ecf 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,6 +1,5 @@ mod attach_worker; mod events; -pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; @@ -8,7 +7,6 @@ mod spawn; mod stacks; mod tracker; -pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 13ed9145..b44afb44 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,17 +1,16 @@ use crate::ebpf::attach_worker::AttachWorker; -use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; use crate::ebpf::stacks::config::stack_capture_from_env; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; -use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; @@ -49,9 +48,14 @@ pub struct Tracker { /// The dedup gate spans the whole BPF object, so a second session would /// reference stack records the first one already consumed. stacks_polled: Option, - /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the - /// session is dropped, so the poller's final drain is included. - mapping_rx: Mutex>>, + /// Number of native perf mapping records lost due to ring-buffer overflow. + mapping_lost: Arc, +} + +fn kill_and_wait(child: &mut std::process::Child) { + // Cleanup is best effort so the setup error remains the returned error. + let _ = child.kill(); + let _ = child.wait(); } impl Tracker { @@ -63,19 +67,9 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let mappings = MappingSupport::detect(); - - // Stacks are raw addresses: without mapped module paths nothing can - // attribute them, so capturing them would only inflate the artifact. - let capture_stacks = match (options.stack_capture, mappings) { - (true, MappingSupport::Unsupported) => { - warn!("Allocation stack capture needs in-kernel path resolution; disabling it"); - false - } - (capture_stacks, _) => capture_stacks, - }; + let capture_stacks = options.stack_capture; Self::build( - MemtrackBpf::new_with_rmap(options.rmap, capture_stacks, mappings)?, + MemtrackBpf::new_with_rmap(options.rmap, capture_stacks)?, options.allocators, capture_stacks, ) @@ -85,7 +79,7 @@ impl Tracker { pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = TrackerOptions::from_env().rmap; Self::build( - MemtrackBpf::with_variant(variant, track_rmap, false, MappingSupport::detect())?, + MemtrackBpf::with_variant(variant, track_rmap, false)?, true, false, ) @@ -100,7 +94,6 @@ impl Tracker { bpf.attach_tracepoints()?; if allocators { bpf.attach_exec_watcher()?; - bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -115,7 +108,7 @@ impl Tracker { worker: Mutex::new(worker), allocators, stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), - mapping_rx: Mutex::new(None), + mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -141,67 +134,53 @@ impl Tracker { wrapped.uid(uid).gid(gid); } - let child = spawn_stopped(&mut wrapped)?; + let mut child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - match self.worker.lock().as_ref() { - Some(worker) => worker.set_root_pid(pid), - // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), - None => {} - } + let setup = (|| -> Result<_> { + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.allocators => bail!("tracker already finished"), + None => {} + } - let (tx, rx) = mpsc::channel(); - let (mapping_tx, mapping_rx) = mpsc::channel(); - let (poller, stack_poller, mapping_poller) = { - let mut bpf = self.bpf.lock(); - bpf.add_tracked_pid(pid)?; - let stack_poller = capture_stacks - .then(|| bpf.poll_stacks(10, tx.clone())) - .transpose()?; - let mapping_poller = bpf - .records_mappings() - .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + let (tx, rx) = mpsc::channel(); + let (poller, stack_poller) = { + let mut bpf = self.bpf.lock(); + bpf.add_tracked_pid(pid)?; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx.clone())?, stack_poller) + }; + let perf_mapping_poller = capture_stacks + .then(|| PerfMappingPoller::start(pid, tx, self.mapping_lost.clone())) .transpose()?; - ( - bpf.poll_events_with_channel(10, tx)?, - stack_poller, - mapping_poller, - ) + + Ok((rx, poller, stack_poller, perf_mapping_poller)) + })(); + let (rx, poller, stack_poller, perf_mapping_poller) = match setup { + Ok(pollers) => pollers, + Err(error) => { + kill_and_wait(&mut child); + return Err(error); + } }; - *self.mapping_rx.lock() = Some(mapping_rx); - resume(pid)?; + + if let Err(error) = resume(pid) { + kill_and_wait(&mut child); + return Err(error); + } Ok(Session::new( child, rx, poller, stack_poller, - mapping_poller, + perf_mapping_poller, )) } - - /// The module mappings recorded during the run, joined with the paths the - /// kernel resolved for them. Call after dropping the session so the poller's - /// final drain is included, and before the BPF object is torn down. - pub fn mappings(&self) -> Result { - let Some(rx) = self.mapping_rx.lock().take() else { - return Ok(MemtrackMappings::default()); - }; - - let records: Vec<_> = rx.try_iter().collect(); - let paths = self.bpf.lock().mapped_paths()?; - - let dropped = self.bpf.lock().mapping_dropped_count()?; - if dropped > 0 { - warn!("{dropped} mapping records were dropped; some modules may be unresolved"); - } - - Ok(MemtrackMappings { - mappings: resolve_mappings(records, &paths), - }) - } - /// Enable allocator-event tracking in the BPF program. Lifetime events /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids /// regardless of this toggle. @@ -217,7 +196,7 @@ impl Tracker { /// Number of events the kernel dropped because the ring buffer was full. /// A non-zero value means the resulting trace is incomplete. pub fn dropped_events_count(&self) -> Result { - self.bpf.lock().dropped_events_count() + Ok(self.bpf.lock().dropped_events_count()? + self.mapping_lost.load(Ordering::Relaxed)) } /// Per-cause counts of stack captures that were skipped or truncated. diff --git a/crates/memtrack/src/lib.rs b/crates/memtrack/src/lib.rs index 1d3278d9..d8cd5f40 100644 --- a/crates/memtrack/src/lib.rs +++ b/crates/memtrack/src/lib.rs @@ -4,6 +4,8 @@ mod bpf_token; mod ebpf; mod ipc; mod kernel; +#[cfg(feature = "ebpf")] +mod perf_mappings; pub mod prelude; #[cfg(feature = "ebpf")] mod session; diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 30b9567e..283cff19 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,11 +159,6 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; - // Collect after the poller drains and before the BPF maps are torn down. - let mappings = tracker.mappings().context("Failed to collect mappings")?; - info!("Recorded {} module mappings", mappings.mappings.len()); - mappings.save_with_pid_to(out_dir, root_pid)?; - // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs new file mode 100644 index 00000000..bc03d39b --- /dev/null +++ b/crates/memtrack/src/perf_mappings.rs @@ -0,0 +1,511 @@ +use crate::prelude::*; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::io; +use std::mem; +use std::os::fd::RawFd; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::JoinHandle; +use std::time::Duration; + +const PERF_TYPE_SOFTWARE: u32 = 1; +const PERF_COUNT_SW_DUMMY: u64 = 9; +const PERF_FLAG_FD_CLOEXEC: libc::c_ulong = 1 << 3; +const PERF_RECORD_LOST: u32 = 2; +const PERF_RECORD_MMAP2: u32 = 10; +const PERF_SAMPLE_TID: u64 = 1 << 1; +const PERF_SAMPLE_TIME: u64 = 1 << 2; +const PERF_EVENT_IOC_ENABLE: libc::c_ulong = 0x2400; +const PERF_EVENT_IOC_DISABLE: libc::c_ulong = 0x2401; +const DATA_PAGES: usize = 64; +const PERF_HEADER_SIZE: usize = 8; + +const fn attr_flag(bit: u32) -> u64 { + #[cfg(target_endian = "little")] + { + 1 << bit + } + #[cfg(target_endian = "big")] + { + (1 << 63) >> bit + } +} + +const PERF_ATTR_DISABLED: u64 = attr_flag(0); +const PERF_ATTR_INHERIT: u64 = attr_flag(1); +const PERF_ATTR_MMAP: u64 = attr_flag(8); +const PERF_ATTR_SAMPLE_ID_ALL: u64 = attr_flag(18); +const PERF_ATTR_MMAP2: u64 = attr_flag(23); +const PERF_ATTR_USE_CLOCKID: u64 = attr_flag(25); + +#[repr(C)] +struct PerfEventAttr { + kind: u32, + size: u32, + config: u64, + sample_period: u64, + sample_type: u64, + read_format: u64, + flags: u64, + wakeup_events: u32, + bp_type: u32, + config1: u64, + config2: u64, + branch_sample_type: u64, + sample_regs_user: u64, + sample_stack_user: u32, + clock_id: i32, +} + +#[repr(C)] +struct PerfEventMmapPage { + version: u32, + compat_version: u32, + lock: u32, + index: u32, + offset: i64, + time_enabled: u64, + time_running: u64, + capabilities: u64, + pmc_width: u16, + time_shift: u16, + time_mult: u32, + time_offset: u64, + time_zero: u64, + size: u32, + reserved: [u8; 118 * 8 + 4], + data_head: u64, + data_tail: u64, + data_offset: u64, + data_size: u64, + aux_head: u64, + aux_tail: u64, + aux_offset: u64, + aux_size: u64, +} + +struct PerfRing { + fd: RawFd, + mapping: *mut u8, + mapping_len: usize, + data_offset: usize, + data_size: usize, + enabled: bool, +} + +// The mapping is exclusively consumed by the poll thread. +unsafe impl Send for PerfRing {} + +impl PerfRing { + fn open(pid: libc::pid_t, cpu: u32, page_size: usize) -> Result { + let mapping_len = page_size + .checked_mul(DATA_PAGES + 1) + .context("perf ring mapping size overflow")?; + ensure!( + mapping_len >= mem::size_of::(), + "perf ring mapping is smaller than its metadata page" + ); + let attr = PerfEventAttr { + kind: PERF_TYPE_SOFTWARE, + size: mem::size_of::() as u32, + config: PERF_COUNT_SW_DUMMY, + sample_period: 0, + sample_type: PERF_SAMPLE_TID | PERF_SAMPLE_TIME, + // PERF_FORMAT_LOST cannot account for inherited child events from this + // parent fd, so PERF_RECORD_LOST remains the complete loss signal. + read_format: 0, + flags: PERF_ATTR_DISABLED + | PERF_ATTR_INHERIT + | PERF_ATTR_MMAP + | PERF_ATTR_SAMPLE_ID_ALL + | PERF_ATTR_MMAP2 + | PERF_ATTR_USE_CLOCKID, + wakeup_events: 1, + bp_type: 0, + config1: 0, + config2: 0, + branch_sample_type: 0, + sample_regs_user: 0, + sample_stack_user: 0, + clock_id: libc::CLOCK_MONOTONIC, + }; + + let fd = unsafe { + libc::syscall( + libc::SYS_perf_event_open, + &attr as *const PerfEventAttr, + pid, + cpu as libc::c_int, + -1, + PERF_FLAG_FD_CLOEXEC, + ) as RawFd + }; + if fd < 0 { + return Err(io::Error::last_os_error()) + .with_context(|| format!("perf_event_open failed for pid {pid} on CPU {cpu}")); + } + + let mapping = unsafe { + libc::mmap( + ptr::null_mut(), + mapping_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + let error = io::Error::last_os_error(); + unsafe { libc::close(fd) }; + return Err(error).context("failed to mmap perf mapping-event ring buffer"); + } + + let page = unsafe { &*(mapping.cast::()) }; + let data_offset = match usize::try_from(page.data_offset) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data offset"); + } + }; + let data_size = match usize::try_from(page.data_size) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data size"); + } + }; + let ring = Self { + fd, + mapping: mapping.cast(), + mapping_len, + data_offset, + data_size, + enabled: false, + }; + + ensure!( + data_offset >= page_size && data_offset % page_size == 0, + "kernel returned an invalid perf ring data offset" + ); + ensure!( + data_size >= PERF_HEADER_SIZE + && data_size % page_size == 0 + && data_size.is_power_of_two(), + "kernel returned an invalid perf ring data size" + ); + let data_end = data_offset + .checked_add(data_size) + .context("perf ring data range overflow")?; + ensure!( + data_end <= mapping_len, + "kernel returned a perf ring outside the mapped area" + ); + + Ok(ring) + } + + fn enable(&mut self) -> Result<()> { + if unsafe { libc::ioctl(self.fd, PERF_EVENT_IOC_ENABLE, 0) } < 0 { + return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); + } + self.enabled = true; + Ok(()) + } + + fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { + let page = unsafe { &mut *(self.mapping.cast::()) }; + let head = unsafe { ptr::read_volatile(&page.data_head) }; + std::sync::atomic::fence(Ordering::Acquire); + let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; + let available = head.wrapping_sub(tail); + + // Once the producer has lapped the consumer, the beginning of the + // stream no longer has a record boundary. Skip the corrupt prefix and + // let the kernel's PERF_RECORD_LOST record account for normal overflow. + if available > self.data_size as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + } else { + while tail != head { + let available = head.wrapping_sub(tail); + if available < PERF_HEADER_SIZE as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let header = self.copy_from_ring(tail, PERF_HEADER_SIZE); + let size = u16::from_ne_bytes([header[6], header[7]]) as usize; + if !(PERF_HEADER_SIZE..=self.data_size).contains(&size) || size as u64 > available { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let record = self.copy_from_ring(tail, size); + self.handle_record(&record, mappings, lost); + tail = tail.wrapping_add(size as u64); + } + } + + std::sync::atomic::fence(Ordering::Release); + unsafe { ptr::write_volatile(&mut page.data_tail, tail) }; + } + + fn copy_from_ring(&self, offset: u64, len: usize) -> Vec { + debug_assert!(len <= self.data_size); + let start = offset as usize & (self.data_size - 1); + let first_len = len.min(self.data_size - start); + let data = unsafe { self.mapping.add(self.data_offset) }; + let mut out = Vec::with_capacity(len); + unsafe { + out.extend_from_slice(std::slice::from_raw_parts(data.add(start), first_len)); + if first_len < len { + out.extend_from_slice(std::slice::from_raw_parts(data, len - first_len)); + } + } + out + } + + fn handle_record(&self, record: &[u8], mappings: &mut Vec, lost: &AtomicU64) { + match read_u32(record, 0) { + Some(PERF_RECORD_MMAP2) => { + if let Some(event) = parse_mmap2(record) { + mappings.push(event); + } + } + Some(PERF_RECORD_LOST) => match read_u64(record, 16) { + Some(count) => { + lost.fetch_add(count, Ordering::Relaxed); + } + None => { + lost.fetch_add(1, Ordering::Relaxed); + } + }, + _ => {} + } + } +} + +impl Drop for PerfRing { + fn drop(&mut self) { + unsafe { + if self.enabled { + let _ = libc::ioctl(self.fd, PERF_EVENT_IOC_DISABLE, 0); + } + libc::munmap(self.mapping.cast(), self.mapping_len); + libc::close(self.fd); + } + } +} + +pub(crate) struct PerfMappingPoller { + ctl: Option>>, + thread: Option>, +} + +impl PerfMappingPoller { + pub(crate) fn start( + pid: libc::pid_t, + tx: Sender, + lost: Arc, + ) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "failed to read the system page size"); + + let cpus = online_cpus()?; + ensure!(!cpus.is_empty(), "no online CPUs reported by the kernel"); + let mut rings = Vec::with_capacity(cpus.len()); + for cpu in cpus { + rings.push(PerfRing::open(pid, cpu, page_size as usize)?); + } + for ring in &mut rings { + ring.enable()?; + } + + let (ctl, ctl_rx) = mpsc::channel::>(); + let thread = std::thread::spawn(move || { + let mut mappings = Vec::new(); + loop { + match ctl_rx.recv_timeout(Duration::from_millis(10)) { + Ok(ack) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + let _ = ack.send(()); + } + Err(RecvTimeoutError::Timeout) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + } + Err(RecvTimeoutError::Disconnected) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + mappings.sort_unstable_by_key(|event| (event.pid, event.timestamp)); + for mapping in mappings { + let _ = tx.send(mapping); + } + break; + } + } + } + }); + + Ok(Self { + ctl: Some(ctl), + thread: Some(thread), + }) + } +} + +impl Drop for PerfMappingPoller { + fn drop(&mut self) { + drop(self.ctl.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn parse_mmap2(record: &[u8]) -> Option { + const FIXED_END: usize = 72; + const SAMPLE_ID_SIZE: usize = 16; + if record.len() < FIXED_END + SAMPLE_ID_SIZE + || read_u32(record, 0)? != PERF_RECORD_MMAP2 + || read_u16(record, 6)? as usize != record.len() + { + return None; + } + + let prot = read_u32(record, 64)?; + if prot & libc::PROT_EXEC as u32 == 0 { + return None; + } + + let path_end = record.len() - SAMPLE_ID_SIZE; + let path_bytes = &record[FIXED_END..path_end]; + let nul = path_bytes.iter().position(|byte| *byte == 0)?; + let path = std::str::from_utf8(&path_bytes[..nul]).ok()?; + if !path.starts_with('/') { + return None; + } + + let major = read_u32(record, 40)? as u64; + let minor = read_u32(record, 44)? as u64; + Some(MemtrackEvent { + pid: read_u32(record, 8)? as libc::pid_t, + tid: read_u32(record, 12)? as libc::pid_t, + timestamp: read_u64(record, record.len() - 8)?, + addr: read_u64(record, 16)?, + kind: MemtrackEventKind::Mapping { + path: path.to_owned(), + dev: (major << 20) | minor, + ino: read_u64(record, 48)?, + file_offset: read_u64(record, 32)?, + len: read_u64(record, 24)?, + }, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_ne_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_ne_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_ne_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn online_cpus() -> Result> { + let spec = std::fs::read_to_string("/sys/devices/system/cpu/online") + .context("failed to read online CPUs")?; + parse_cpu_list(spec.trim()) +} + +fn parse_cpu_list(spec: &str) -> Result> { + let mut cpus = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + ensure!(!part.is_empty(), "invalid empty CPU range"); + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start.parse::()?, end.parse::()?), + None => { + let cpu = part.parse::()?; + (cpu, cpu) + } + }; + ensure!(start <= end, "invalid CPU range {part}"); + cpus.extend(start..=end); + } + Ok(cpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_cpu_ranges() { + assert_eq!(parse_cpu_list("0-2,5,8-9").unwrap(), vec![0, 1, 2, 5, 8, 9]); + } + + #[test] + fn parses_executable_mmap2() { + let path = b"/tmp/module.so\0"; + let mut record = vec![0; 72 + path.len() + 16]; + record[0..4].copy_from_slice(&PERF_RECORD_MMAP2.to_ne_bytes()); + let size = record.len() as u16; + record[6..8].copy_from_slice(&size.to_ne_bytes()); + record[8..12].copy_from_slice(&7_u32.to_ne_bytes()); + record[12..16].copy_from_slice(&8_u32.to_ne_bytes()); + record[16..24].copy_from_slice(&0x4000_u64.to_ne_bytes()); + record[24..32].copy_from_slice(&0x2000_u64.to_ne_bytes()); + record[32..40].copy_from_slice(&0x1000_u64.to_ne_bytes()); + record[40..44].copy_from_slice(&1_u32.to_ne_bytes()); + record[44..48].copy_from_slice(&2_u32.to_ne_bytes()); + record[48..56].copy_from_slice(&42_u64.to_ne_bytes()); + record[64..68].copy_from_slice(&(libc::PROT_EXEC as u32).to_ne_bytes()); + record[72..72 + path.len()].copy_from_slice(path); + let timestamp = 99_u64; + let time_offset = record.len() - 8; + record[time_offset..].copy_from_slice(×tamp.to_ne_bytes()); + + assert_eq!( + parse_mmap2(&record), + Some(MemtrackEvent { + pid: 7, + tid: 8, + timestamp, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: "/tmp/module.so".into(), + dev: (1 << 20) | 2, + ino: 42, + file_offset: 0x1000, + len: 0x2000, + }, + }) + ); + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 551db639..c8f7ef47 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,5 @@ use crate::ebpf::poller::RingBufferPoller; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; @@ -9,9 +10,15 @@ use std::sync::mpsc::Receiver; pub struct Session { child: Child, events: Option>, + + // Drop order is part of the artifact compatibility contract. Rust drops + // fields in declaration order: both BPF pollers must stay before the perf + // mapping poller. Their Drop implementations disconnect, fully drain, and + // join their poll threads before PerfMappingPoller drops and emits its + // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, _stack_poller: Option, - _mapping_poller: Option, + _perf_mapping_poller: Option, } impl Session { @@ -20,14 +27,14 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, - mapping_poller: Option, + perf_mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, - _mapping_poller: mapping_poller, + _perf_mapping_poller: perf_mapping_poller, } } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 28950dd7..d3975b6e 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -18,13 +18,6 @@ fn compile_fixture( temp_dir.path(), ) } -fn require_mapping_support() -> bool { - if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { - eprintln!("skipping stack capture test: mapping support is unavailable"); - return false; - } - true -} /// The stack identity carried by each allocation and deallocation event that has one. fn event_hashes(events: &[MemtrackEvent]) -> Vec { @@ -54,9 +47,6 @@ fn record_hashes(events: &[MemtrackEvent]) -> HashSet { #[test_with::env(GITHUB_ACTIONS)] #[test_log::test] fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } let temp_dir = TempDir::new()?; let binary = compile_fixture("stack_paths", &temp_dir)?; let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; @@ -127,9 +117,6 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } let temp_dir = TempDir::new()?; let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; @@ -206,9 +193,6 @@ fn explicit_disable_suppresses_stack_capture() -> Result<(), Box Result<(), Box> { - if !require_mapping_support() { - return Ok(()); - } for (name, source) in [ ( "nested_doubling", diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs index b271dfee..41bf8428 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mappings.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mappings.rs @@ -2,19 +2,6 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::ops::Range; -/// The file-backed mappings the tracked process tree loaded, recorded as they -/// happened. Companion to the event stream: allocation stacks are raw -/// addresses, and these are what turns them back into modules. -/// -/// Kept out of the event stream so a consumer that only needs the module set -/// does not have to decode millions of allocation events. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MemtrackMappings { - pub mappings: Vec, -} - -impl super::super::ArtifactExt for MemtrackMappings {} - /// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` /// would describe it. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 0ba0ced4..a637c5a6 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -104,6 +104,15 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + /// One executable file mapping from a native PERF_RECORD_MMAP2 record. + /// The common event header carries its address, process, and timestamp. + Mapping { + path: String, + dev: u64, + ino: u64, + file_offset: u64, + len: u64, + }, Stack { #[serde(flatten)] @@ -180,6 +189,19 @@ mod tests { size: 40960, }, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 400, + addr: 0x400000, + kind: MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + }, ]; let artifact = MemtrackArtifact { @@ -239,6 +261,13 @@ mod tests { MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, MemtrackEventKind::Stack { record: Box::new(StackRecord { hash: 0xDEAD_BEEF, diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index f8dcd0e9..cd34a6cb 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -3,7 +3,7 @@ use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; -use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind, ProcessMapping}; use runner_shared::metadata::MemtrackMetadata; use std::collections::HashMap; use std::os::unix::fs::MetadataExt; @@ -47,7 +47,7 @@ pub fn save_module_artifacts( /// Read every mapping artifact in the folder. One is written per tracked root /// process, so a run with several of them contributes several files. fn read_mappings(results_folder: &Path) -> Result> { - let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + let suffix = format!(".{}.msgpack", MemtrackArtifact::name()); let mut mappings = Vec::new(); for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { @@ -56,10 +56,47 @@ fn read_mappings(results_folder: &Path) -> Result> { } let file = std::fs::File::open(entry.path())?; - let artifact = MemtrackMappings::decode_from_reader(file) - .with_context(|| format!("Failed to decode {:?}", entry.path()))?; - mappings.extend(artifact.mappings); + mappings.extend( + read_mappings_from_artifact(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?, + ); + } + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); + Ok(mappings) +} + +fn read_mappings_from_artifact(reader: R) -> Result> { + let stream = MemtrackArtifact::decode_streamed(reader)?; + let mut mappings = Vec::new(); + + for event in stream { + let MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } = event.kind + else { + continue; + }; + + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + mappings.push(ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + }); } + Ok(mappings) } @@ -161,6 +198,7 @@ fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { #[cfg(all(test, target_os = "linux"))] mod tests { use super::*; + use runner_shared::artifacts::MemtrackEvent; fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { ProcessMapping { @@ -216,7 +254,166 @@ mod tests { } #[test] - fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + fn sorts_interleaved_mapping_artifacts_by_pid_and_timestamp() { + let results = tempfile::tempdir().unwrap(); + + // Separate files model records drained from different per-CPU rings. + MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 20, + addr: 0x2000, + kind: MemtrackEventKind::Mapping { + path: "second-module.so".to_string(), + dev: 2, + ino: 2, + file_offset: 0x2000, + len: 0x1000, + }, + }, + MemtrackEvent { + pid: 7, + tid: 7, + timestamp: 30, + addr: 0x7000, + kind: MemtrackEventKind::Mapping { + path: "child-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0x3000, + len: 0x1000, + }, + }, + ], + } + .save_file_to(results.path(), "cpu1.MemtrackArtifact.msgpack") + .unwrap(); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 10, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: "first-module.so".to_string(), + dev: 1, + ino: 1, + file_offset: 0x1000, + len: 0x1000, + }, + }], + } + .save_file_to(results.path(), "cpu0.MemtrackArtifact.msgpack") + .unwrap(); + + let mappings = read_mappings(results.path()).unwrap(); + assert_eq!( + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.timestamp)) + .collect::>(), + vec![(7, 30), (42, 10), (42, 20)] + ); + assert_eq!(mappings[0].path, "child-module.so"); + assert_eq!(mappings[1].path, "first-module.so"); + assert_eq!(mappings[2].path, "second-module.so"); + } + #[test] + fn extracts_all_mapping_events_from_a_streamed_artifact() { + const FIRST_MODULE: &str = "first-module.so"; + const SECOND_MODULE: &str = "second-module.so"; + let artifact = MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 10, + addr: 0, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, + }, + MemtrackEvent { + pid: 7, + tid: 8, + timestamp: 11, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + len: 0x2000, + }, + }, + MemtrackEvent { + pid: 9, + tid: 10, + timestamp: 12, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + len: 0x3000, + }, + }, + MemtrackEvent { + pid: 99, + tid: 99, + timestamp: 99, + addr: u64::MAX, + kind: MemtrackEventKind::Mapping { + path: "overflow-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0, + len: 1, + }, + }, + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 13, + addr: 0x2000, + kind: MemtrackEventKind::Free { stack_hash: 0 }, + }, + ], + }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + + assert_eq!( + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap(), + vec![ + ProcessMapping { + pid: 7, + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + avma_range: 0x1000..0x3000, + timestamp: 11, + }, + ProcessMapping { + pid: 9, + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + avma_range: 0x4000..0x7000, + timestamp: 12, + }, + ] + ); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_streamed_mapping() { const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; let profile = tempfile::tempdir().unwrap(); @@ -224,15 +421,19 @@ mod tests { std::fs::create_dir_all(&results).unwrap(); let (dev, ino) = s_dev_of(MODULE); - MemtrackMappings { - mappings: vec![ProcessMapping { + MemtrackArtifact { + events: vec![MemtrackEvent { pid: 1234, - path: MODULE.to_string(), - dev, - ino, - file_offset: 0x5_2000, - avma_range: 0x5555_555a_7000..0x5555_556b_0000, + tid: 1234, timestamp: 999, + addr: 0x5555_555a_7000, + kind: MemtrackEventKind::Mapping { + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + len: 0x109_000, + }, }], } .save_with_pid_to(&results, 1234) From 01a1bbf19179e8b0ce04cc82bc23110187e2697a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 20:42:54 +0200 Subject: [PATCH 18/27] fixup! docs(memtrack): describe the mapping recorder --- crates/memtrack/AGENTS.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 819eb584..2439bf0f 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,13 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. -Mapping recorder (`src/ebpf/c/mappings.bpf.h`, `src/ebpf/mappings/`): an LSM program on `mmap_file` resolves each mapped file's path once per inode into `path_by_inode` (`bpf_path_d_path` on kernels >= 6.12, `bpf_d_path` on the sleepable hook from 5.11), and an `fentry/perf_event_mmap` program emits executable-mapping geometry (`dev`/`ino`/`file_offset`/`start`/`end`) on the `mappings` ring buffer. Userspace joins the two into a `MemtrackMappings` artifact, which the runner turns into `unwind_data`/`symbols.map` files and a `memtrack.metadata` so allocation stacks unwind off-box. `MappingSupport::detect()` gates the programs on the kernel release **and** `bpf` being in `/sys/kernel/security/lsm`; with neither available, stack capture is disabled since nothing could attribute the stacks. +Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.` + +### Event stream compatibility + +Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream. + +This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown. > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `mappings/` (records/resolve/support), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/mappings.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). From b18daca2399f3c2268b5244d566c46001ab6db03 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 1 Sep 2026 20:55:42 +0200 Subject: [PATCH 19/27] fixup! refactor(memtrack): capture module mappings with perf --- crates/memtrack/tests/shared.rs | 4 ++++ crates/memtrack/tests/stack_tests.rs | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 9bc2e2e9..8e70c773 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -180,6 +180,10 @@ pub fn track_binary(binary: &Path) -> TrackResult { track_command(Command::new(binary)) } +pub fn track_binary_with_env(binary: &Path) -> TrackResult { + track_command_with_tracker(Command::new(binary), Tracker::new()?) +} + pub fn compile_c_source( source_code: &str, name: &str, diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index d3975b6e..5dffa3a9 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -162,7 +162,7 @@ fn explicit_disable_suppresses_stack_capture() -> Result<(), Box Date: Tue, 1 Sep 2026 21:29:48 +0200 Subject: [PATCH 20/27] fixup! feat(runner): write memtrack module artifacts and metadata --- src/executor/memory/module_artifacts.rs | 256 +++++++++++++++++++++--- 1 file changed, 232 insertions(+), 24 deletions(-) diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index cd34a6cb..1df0c7cc 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -3,6 +3,7 @@ use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; +use libc::pid_t; use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind, ProcessMapping}; use runner_shared::metadata::MemtrackMetadata; use std::collections::HashMap; @@ -11,6 +12,32 @@ use std::path::{Path, PathBuf}; const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; +enum MappingTimelineEvent { + Mapping(ProcessMapping), + Fork { + parent_pid: pid_t, + child_pid: pid_t, + timestamp: u64, + }, + Exec { + pid: pid_t, + timestamp: u64, + }, +} + +impl MappingTimelineEvent { + /// Ties break so that an exec purges before the mappings recorded at that + /// same instant, and a fork inherits everything mapped up to and including + /// its own instant. + fn order(&self) -> (u64, u8) { + match self { + Self::Exec { timestamp, .. } => (*timestamp, 0), + Self::Mapping(mapping) => (mapping.timestamp, 1), + Self::Fork { timestamp, .. } => (*timestamp, 2), + } + } +} + /// Turn the mappings memtrack recorded into the artifacts an offline unwinder /// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the /// `memtrack.metadata` referencing them per pid. @@ -65,36 +92,88 @@ fn read_mappings(results_folder: &Path) -> Result> { Ok(mappings) } +/// Reconstruct mappings across forks because inherited perf events do not +/// synthesize mappings that already existed when a child was forked. fn read_mappings_from_artifact(reader: R) -> Result> { let stream = MemtrackArtifact::decode_streamed(reader)?; - let mut mappings = Vec::new(); + let mut timeline = Vec::new(); for event in stream { - let MemtrackEventKind::Mapping { - path, - dev, - ino, - file_offset, - len, - } = event.kind - else { - continue; - }; + match event.kind { + MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } => { + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + timeline.push(MappingTimelineEvent::Mapping(ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + })); + } + MemtrackEventKind::Fork { parent_pid } => { + timeline.push(MappingTimelineEvent::Fork { + parent_pid, + child_pid: event.pid, + timestamp: event.timestamp, + }); + } + MemtrackEventKind::Exec => { + timeline.push(MappingTimelineEvent::Exec { + pid: event.pid, + timestamp: event.timestamp, + }); + } + _ => {} + } + } - let Some(end) = event.addr.checked_add(len) else { - debug!("Skipping mapping for {path}: address range overflows"); - continue; - }; + timeline.sort_by_key(|event| event.order()); - mappings.push(ProcessMapping { - pid: event.pid, - path, - dev, - ino, - file_offset, - avma_range: event.addr..end, - timestamp: event.timestamp, - }); + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); + + for event in timeline { + match event { + MappingTimelineEvent::Mapping(mapping) => { + live_mappings + .entry(mapping.pid) + .or_default() + .push(mapping.clone()); + mappings.push(mapping); + } + MappingTimelineEvent::Fork { + parent_pid, + child_pid, + timestamp, + } => { + let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); + let child_mappings = inherited + .into_iter() + .map(|mut mapping| { + mapping.pid = child_pid; + mapping.timestamp = timestamp; + mapping + }) + .collect::>(); + mappings.extend(child_mappings.iter().cloned()); + live_mappings.insert(child_pid, child_mappings); + } + MappingTimelineEvent::Exec { pid, .. } => { + live_mappings.remove(&pid); + } + } } Ok(mappings) @@ -470,4 +549,133 @@ mod tests { PathBuf::from(MODULE) ); } + + fn mapping_event(pid: pid_t, timestamp: u64, addr: u64, path: &str) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr, + kind: MemtrackEventKind::Mapping { + path: path.to_string(), + dev: 1, + ino: 1, + file_offset: 0, + len: 0x1000, + }, + } + } + + fn fork_event(child_pid: pid_t, parent_pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid: child_pid, + tid: child_pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Fork { parent_pid }, + } + } + + fn exec_event(pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Exec, + } + } + + fn decode_lifecycle(events: Vec) -> Vec { + let artifact = MemtrackArtifact { events }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap() + } + + fn mapping_summary(mappings: &[ProcessMapping]) -> Vec<(pid_t, &str, u64)> { + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.path.as_str(), mapping.timestamp)) + .collect() + } + + #[test] + fn fork_without_exec_inherits_only_mappings_before_fork() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + mapping_event(100, 30, 0x2000, "after-fork.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 30), + ] + ); + } + + #[test] + fn exec_stops_inheriting_parent_mappings() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + exec_event(200, 30), + mapping_event(100, 40, 0x2000, "after-fork.so"), + mapping_event(200, 50, 0x3000, "after-exec.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 40), + (200, "after-exec.so", 50), + ] + ); + } + + #[test] + fn grandchild_inherits_transitively_from_forked_child() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "root.so"), + fork_event(200, 100, 20), + mapping_event(200, 25, 0x2000, "child.so"), + fork_event(300, 200, 30), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "root.so", 10), + (200, "root.so", 20), + (200, "child.so", 25), + (300, "root.so", 30), + (300, "child.so", 30), + ] + ); + } + + #[test] + fn equal_timestamp_events_follow_exec_mapping_fork_rank() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-exec.so"), + fork_event(200, 100, 20), + mapping_event(100, 20, 0x2000, "after-exec.so"), + exec_event(100, 20), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-exec.so", 10), + (100, "after-exec.so", 20), + (200, "after-exec.so", 20), + ] + ); + } } From d27431404bc2f8d79a4ea964e88238eaffa35e38 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 21/27] fixup! feat(memtrack): capture allocation stacks in eBPF --- .../memtrack/src/ebpf/c/stack_capture.bpf.h | 57 +++++++------------ .../src/artifacts/memtrack/mod.rs | 1 + 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h index ca9eda35..3ac1b1cb 100644 --- a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -5,27 +5,22 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -/* At allocator entry the caller's raw user stack is copied and hashed; the hash - * travels on the allocation event as its stack identity. The first time a hash - * is seen, the copied bytes plus a register snapshot are emitted as a stack - * record so the stack can be DWARF-unwound offline, with an in-kernel - * frame-pointer walk alongside as the fallback. +/* Raw user-stack bytes and registers are emitted once per hash for offline + * DWARF unwinding. Allocation events carry the hash; bpf_get_stackid() supplies + * the frame-pointer fallback. * - * Hashing raw bytes rather than unwound frames splits one call path into - * several identities whenever locals or arguments in the copied region differ. - * That only costs extra records; the reverse trade (aliasing) would corrupt - * attribution. + * Stack data changes may give one call path multiple hashes. */ const volatile __u8 capture_stacks_enabled = 0; #define STACK_TRACE_MAX_DEPTH 127 -/* Copy granularity: a recovered length is exact only to within one chunk. */ +/* Captured lengths are rounded down to this granularity. */ #define STACK_COPY_CHUNK 512 #define FNV64_OFFSET 0xcbf29ce484222325ULL #define FNV64_PRIME 0x00000100000001b3ULL -/* Frame-pointer walk results, indexed by the id bpf_get_stackid() returns. */ +/* Frame-pointer fallback keyed by bpf_get_stackid(). */ struct { __uint(type, BPF_MAP_TYPE_STACK_TRACE); __uint(max_entries, 16384); @@ -33,16 +28,13 @@ struct { __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); } stack_traces SEC(".maps"); -/* Records are bulky but rare (one per distinct hash), so they get their own - * ring rather than inflating the fixed-size `struct event` path. */ +/* A separate ring keeps allocation events fixed-size. */ BPF_RINGBUF(stacks, 64 * 1024 * 1024); BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); -/* The record is built in place here and handed to the ring buffer as one - * contiguous variable-length blob. `words` aliases `bytes` so the hash loop - * reads whole registers without a per-byte shift chain. */ +/* The union permits word-wise hashing before emitting a variable-length record. */ struct stack_scratch_buf { struct stack_header header; union { @@ -58,8 +50,7 @@ struct { __type(value, struct stack_scratch_buf); } stack_scratch SEC(".maps"); -/* The value is 64-bit because the BPF backend of older clang cannot select a - * 32-bit atomic compare-and-swap. */ +/* Older clang BPF backends cannot lower a 32-bit atomic compare-and-swap. */ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); @@ -118,14 +109,8 @@ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct tas __u64 sp = PT_REGS_SP(ctx); const __u32 want = 8192; - /* bpf_probe_read_user() is all-or-nothing and the readable region ends at - * the top of the stack mapping, which is not knowable up front, so the copy - * advances in chunks and stops at the first unreadable one. - * - * Each chunk is hashed as it lands, over a constant iteration count the - * compiler fully unrolls. One loop over the whole copy instead costs the - * verifier a state fork per word and blows the one-million instruction - * budget well below the maximum copy size. */ + /* Chunked reads stop at the first unreadable stack region. Fixed-size, + * unrolled hash loops bound verifier state growth. */ __u64 hash = FNV64_OFFSET; __u32 got = 0; #pragma clang loop unroll(disable) @@ -156,9 +141,8 @@ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct tas bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); } - /* Fold in the length so a truncated prefix of a deep stack cannot collide - * with a full copy of a shallower one, and keep 0 reserved as the "no - * stack" marker on allocation events. */ + /* Length distinguishes a full copy from the same bytes as a truncated prefix. + * Zero is reserved for allocation events without a stack. */ hash = (hash ^ got) * FNV64_PRIME; if (hash == 0) { hash = FNV64_OFFSET; @@ -166,12 +150,11 @@ static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct tas __u8 marker = 1; long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); - if (gate_result == -17) { /* -EEXIST: already emitted */ + if (gate_result == -17) { /* -EEXIST */ return hash; } if (gate_result != 0) { - /* A full gate cannot retain this identity, so emit it on every - * occurrence rather than make the allocation hash unresolvable. */ + /* Re-emit when deduplication is full so the hash remains resolvable. */ bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); } @@ -216,8 +199,8 @@ static __always_inline __u64 capture_stack(struct pt_regs* ctx) { if (!busy) { return 0; } - /* uprobe_multi runs programs without the bpf_prog_active recursion guard, - * so a task preempting this one on the same CPU could corrupt the scratch. */ + /* uprobe_multi lacks the bpf_prog_active recursion guard; preemption can + * re-enter on the same per-CPU scratch buffer. */ if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); return 0; @@ -225,9 +208,8 @@ static __always_inline __u64 capture_stack(struct pt_regs* ctx) { __u64 hash = capture_stack_inner(ctx, ids); - /* A plain store, not an atomic release: the only contender is a task that - * preempted this one on this same CPU, and the context switch between them - * already orders the write. The BPF backend cannot select a release store. */ + /* A same-CPU contender runs only after a context switch, which orders this + * plain store before scratch-buffer reuse. */ *busy = 0; return hash; } @@ -241,7 +223,6 @@ static __always_inline void stash_stack_hash(__u64 hash) { bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); } -/* Every return path must clear the per-thread slot. */ static __always_inline __u64 take_stack_hash(void) { if (!capture_stacks_enabled) { return 0; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index a637c5a6..27d1b088 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -115,6 +115,7 @@ pub enum MemtrackEventKind { }, Stack { + // Box keeps the MemtrackEventKind enum small across millions of events. #[serde(flatten)] record: Box, }, From 021dd2fde7f0a6adbb53aaa32d22d01586c04f14 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 22/27] fixup! refactor(memtrack): capture module mappings with perf --- Cargo.lock | 10 ++ Cargo.toml | 1 + crates/memtrack/Cargo.toml | 1 + crates/memtrack/src/perf_mappings.rs | 146 +++++------------- .../src/artifacts/memtrack/mappings.rs | 24 --- .../src/artifacts/memtrack/mod.rs | 2 - 6 files changed, 48 insertions(+), 136 deletions(-) delete mode 100644 crates/runner-shared/src/artifacts/memtrack/mappings.rs diff --git a/Cargo.lock b/Cargo.lock index 8e555476..4956c2d3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2402,6 +2402,7 @@ dependencies = [ "object", "parking_lot", "paste", + "perf-event-open-sys", "rayon", "rstest", "runner-shared", @@ -2813,6 +2814,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-open-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5f8d1487a4ffa23c80a1c355dd27235f9b66fb71ba0f261eb417e4fe8451347" +dependencies = [ + "libc", +] + [[package]] name = "pest" version = "2.8.6" diff --git a/Cargo.toml b/Cargo.toml index 8bd28f03..34be934d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ ipc-channel = "0.20" itertools = "0.14.0" rayon = "1.12" linux-perf-event-reader = "0.10.2" # matches the version linux-perf-data resolves to +perf-event-open-sys = "6.0" env_logger = "0.11.10" tempfile = "3.27.0" object = { version = "0.39", default-features = false, features = ["read_core", "elf"] } diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 4912802d..68da1218 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -34,6 +34,7 @@ itertools = { workspace = true } paste = "1.0.15" libbpf-rs = { version = "0.26", features = ["vendored"], optional = true } object = { workspace = true } +perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs index bc03d39b..45350dbf 100644 --- a/crates/memtrack/src/perf_mappings.rs +++ b/crates/memtrack/src/perf_mappings.rs @@ -1,7 +1,12 @@ use crate::prelude::*; +use perf_event_open_sys::bindings::{ + PERF_COUNT_SW_DUMMY, PERF_FLAG_FD_CLOEXEC, PERF_RECORD_LOST, PERF_RECORD_MMAP2, + PERF_SAMPLE_TID, PERF_SAMPLE_TIME, PERF_TYPE_SOFTWARE, perf_event_attr, perf_event_header, + perf_event_mmap_page, +}; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; use std::io; -use std::mem; +use std::mem::size_of; use std::os::fd::RawFd; use std::ptr; use std::sync::Arc; @@ -10,81 +15,7 @@ use std::sync::mpsc::{self, RecvTimeoutError, Sender}; use std::thread::JoinHandle; use std::time::Duration; -const PERF_TYPE_SOFTWARE: u32 = 1; -const PERF_COUNT_SW_DUMMY: u64 = 9; -const PERF_FLAG_FD_CLOEXEC: libc::c_ulong = 1 << 3; -const PERF_RECORD_LOST: u32 = 2; -const PERF_RECORD_MMAP2: u32 = 10; -const PERF_SAMPLE_TID: u64 = 1 << 1; -const PERF_SAMPLE_TIME: u64 = 1 << 2; -const PERF_EVENT_IOC_ENABLE: libc::c_ulong = 0x2400; -const PERF_EVENT_IOC_DISABLE: libc::c_ulong = 0x2401; const DATA_PAGES: usize = 64; -const PERF_HEADER_SIZE: usize = 8; - -const fn attr_flag(bit: u32) -> u64 { - #[cfg(target_endian = "little")] - { - 1 << bit - } - #[cfg(target_endian = "big")] - { - (1 << 63) >> bit - } -} - -const PERF_ATTR_DISABLED: u64 = attr_flag(0); -const PERF_ATTR_INHERIT: u64 = attr_flag(1); -const PERF_ATTR_MMAP: u64 = attr_flag(8); -const PERF_ATTR_SAMPLE_ID_ALL: u64 = attr_flag(18); -const PERF_ATTR_MMAP2: u64 = attr_flag(23); -const PERF_ATTR_USE_CLOCKID: u64 = attr_flag(25); - -#[repr(C)] -struct PerfEventAttr { - kind: u32, - size: u32, - config: u64, - sample_period: u64, - sample_type: u64, - read_format: u64, - flags: u64, - wakeup_events: u32, - bp_type: u32, - config1: u64, - config2: u64, - branch_sample_type: u64, - sample_regs_user: u64, - sample_stack_user: u32, - clock_id: i32, -} - -#[repr(C)] -struct PerfEventMmapPage { - version: u32, - compat_version: u32, - lock: u32, - index: u32, - offset: i64, - time_enabled: u64, - time_running: u64, - capabilities: u64, - pmc_width: u16, - time_shift: u16, - time_mult: u32, - time_offset: u64, - time_zero: u64, - size: u32, - reserved: [u8; 118 * 8 + 4], - data_head: u64, - data_tail: u64, - data_offset: u64, - data_size: u64, - aux_head: u64, - aux_tail: u64, - aux_offset: u64, - aux_size: u64, -} struct PerfRing { fd: RawFd, @@ -104,43 +35,36 @@ impl PerfRing { .checked_mul(DATA_PAGES + 1) .context("perf ring mapping size overflow")?; ensure!( - mapping_len >= mem::size_of::(), + mapping_len >= size_of::(), "perf ring mapping is smaller than its metadata page" ); - let attr = PerfEventAttr { - kind: PERF_TYPE_SOFTWARE, - size: mem::size_of::() as u32, - config: PERF_COUNT_SW_DUMMY, - sample_period: 0, - sample_type: PERF_SAMPLE_TID | PERF_SAMPLE_TIME, + let mut attr = perf_event_attr { + type_: PERF_TYPE_SOFTWARE, + size: size_of::() as u32, + config: PERF_COUNT_SW_DUMMY as u64, + sample_type: (PERF_SAMPLE_TID | PERF_SAMPLE_TIME) as u64, // PERF_FORMAT_LOST cannot account for inherited child events from this // parent fd, so PERF_RECORD_LOST remains the complete loss signal. read_format: 0, - flags: PERF_ATTR_DISABLED - | PERF_ATTR_INHERIT - | PERF_ATTR_MMAP - | PERF_ATTR_SAMPLE_ID_ALL - | PERF_ATTR_MMAP2 - | PERF_ATTR_USE_CLOCKID, - wakeup_events: 1, - bp_type: 0, - config1: 0, - config2: 0, - branch_sample_type: 0, - sample_regs_user: 0, - sample_stack_user: 0, - clock_id: libc::CLOCK_MONOTONIC, + clockid: libc::CLOCK_MONOTONIC, + ..Default::default() }; + attr.__bindgen_anon_2.wakeup_events = 1; + attr.set_disabled(1); + attr.set_inherit(1); + attr.set_mmap(1); + attr.set_sample_id_all(1); + attr.set_mmap2(1); + attr.set_use_clockid(1); let fd = unsafe { - libc::syscall( - libc::SYS_perf_event_open, - &attr as *const PerfEventAttr, + perf_event_open_sys::perf_event_open( + &mut attr, pid, - cpu as libc::c_int, + cpu as _, -1, - PERF_FLAG_FD_CLOEXEC, - ) as RawFd + PERF_FLAG_FD_CLOEXEC as _, + ) }; if fd < 0 { return Err(io::Error::last_os_error()) @@ -163,7 +87,7 @@ impl PerfRing { return Err(error).context("failed to mmap perf mapping-event ring buffer"); } - let page = unsafe { &*(mapping.cast::()) }; + let page = unsafe { &*(mapping.cast::()) }; let data_offset = match usize::try_from(page.data_offset) { Ok(value) => value, Err(_) => { @@ -198,7 +122,7 @@ impl PerfRing { "kernel returned an invalid perf ring data offset" ); ensure!( - data_size >= PERF_HEADER_SIZE + data_size >= size_of::() && data_size % page_size == 0 && data_size.is_power_of_two(), "kernel returned an invalid perf ring data size" @@ -215,7 +139,7 @@ impl PerfRing { } fn enable(&mut self) -> Result<()> { - if unsafe { libc::ioctl(self.fd, PERF_EVENT_IOC_ENABLE, 0) } < 0 { + if unsafe { perf_event_open_sys::ioctls::ENABLE(self.fd, 0) } < 0 { return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); } self.enabled = true; @@ -223,7 +147,7 @@ impl PerfRing { } fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { - let page = unsafe { &mut *(self.mapping.cast::()) }; + let page = unsafe { &mut *(self.mapping.cast::()) }; let head = unsafe { ptr::read_volatile(&page.data_head) }; std::sync::atomic::fence(Ordering::Acquire); let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; @@ -238,15 +162,17 @@ impl PerfRing { } else { while tail != head { let available = head.wrapping_sub(tail); - if available < PERF_HEADER_SIZE as u64 { + if available < size_of::() as u64 { lost.fetch_add(1, Ordering::Relaxed); tail = head; break; } - let header = self.copy_from_ring(tail, PERF_HEADER_SIZE); + let header = self.copy_from_ring(tail, size_of::()); let size = u16::from_ne_bytes([header[6], header[7]]) as usize; - if !(PERF_HEADER_SIZE..=self.data_size).contains(&size) || size as u64 > available { + if !(size_of::()..=self.data_size).contains(&size) + || size as u64 > available + { lost.fetch_add(1, Ordering::Relaxed); tail = head; break; @@ -301,7 +227,7 @@ impl Drop for PerfRing { fn drop(&mut self) { unsafe { if self.enabled { - let _ = libc::ioctl(self.fd, PERF_EVENT_IOC_DISABLE, 0); + let _ = perf_event_open_sys::ioctls::DISABLE(self.fd, 0); } libc::munmap(self.mapping.cast(), self.mapping_len); libc::close(self.fd); diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs deleted file mode 100644 index 41bf8428..00000000 --- a/crates/runner-shared/src/artifacts/memtrack/mappings.rs +++ /dev/null @@ -1,24 +0,0 @@ -use libc::pid_t; -use serde::{Deserialize, Serialize}; -use std::ops::Range; - -/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` -/// would describe it. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct ProcessMapping { - pub pid: pid_t, - /// Resolved in-kernel at mmap time, so it is correct for the mapping - /// process's mount namespace even if the process is already gone. - pub path: String, - /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at - /// analysis time that the path still names the file that was mapped. - pub dev: u64, - pub ino: u64, - /// Offset of the mapping's first byte in the file. In bytes, matching - /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. - pub file_offset: u64, - pub avma_range: Range, - /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The - /// mapping is valid from here until a later mapping covers the range. - pub timestamp: u64, -} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 27d1b088..fed115f2 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,11 +2,9 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; -mod mappings; mod pipeline; mod writer; -pub use mappings::*; pub use pipeline::*; pub use writer::*; From 7201cfc8759db23be66244e94fc65367ab9a9968 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 23/27] fixup! feat(runner): write memtrack module artifacts and metadata --- crates/runner-shared/src/metadata.rs | 12 +- src/executor/memory/module_artifacts.rs | 176 +++++++++++------------- 2 files changed, 94 insertions(+), 94 deletions(-) diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 654ae298..2e48a2fd 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -126,6 +126,16 @@ pub struct MemtrackMetadata { } impl MemtrackMetadata { + pub const CURRENT_VERSION: u64 = 1; + + pub fn new(integration: (String, String), artifacts: ModuleArtifacts) -> Self { + Self { + version: Self::CURRENT_VERSION, + integration, + artifacts, + } + } + pub fn from_reader(reader: R) -> anyhow::Result { serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") } @@ -233,7 +243,7 @@ mod tests { #[test] fn memtrack_metadata_round_trips() { let metadata = MemtrackMetadata { - version: 1, + version: MemtrackMetadata::CURRENT_VERSION, integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), artifacts: populated_artifacts(), }; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs index 1df0c7cc..e888668b 100644 --- a/src/executor/memory/module_artifacts.rs +++ b/src/executor/memory/module_artifacts.rs @@ -4,38 +4,33 @@ use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; -use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind, ProcessMapping}; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind}; use runner_shared::metadata::MemtrackMetadata; +use runner_shared::unwind_data::ProcessUnwindData; use std::collections::HashMap; +use std::ops::Range; use std::os::unix::fs::MetadataExt; use std::path::{Path, PathBuf}; -const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; - -enum MappingTimelineEvent { - Mapping(ProcessMapping), - Fork { - parent_pid: pid_t, - child_pid: pid_t, - timestamp: u64, - }, - Exec { - pid: pid_t, - timestamp: u64, - }, -} - -impl MappingTimelineEvent { - /// Ties break so that an exec purges before the mappings recorded at that - /// same instant, and a fork inherits everything mapped up to and including - /// its own instant. - fn order(&self) -> (u64, u8) { - match self { - Self::Exec { timestamp, .. } => (*timestamp, 0), - Self::Mapping(mapping) => (mapping.timestamp, 1), - Self::Fork { timestamp, .. } => (*timestamp, 2), - } - } +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProcessMapping { + pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + dev: u64, + ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + file_offset: u64, + avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + timestamp: u64, } /// Turn the mappings memtrack recorded into the artifacts an offline unwinder @@ -63,12 +58,7 @@ pub fn save_module_artifacts( ); let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); - MemtrackMetadata { - version: MEMTRACK_METADATA_CURRENT_VERSION, - integration, - artifacts: saved.artifacts, - } - .save_to(profile_folder) + MemtrackMetadata::new(integration, saved.artifacts).save_to(profile_folder) } /// Read every mapping artifact in the folder. One is written per tracked root @@ -88,6 +78,7 @@ fn read_mappings(results_folder: &Path) -> Result> { .with_context(|| format!("Failed to decode {:?}", entry.path()))?, ); } + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); Ok(mappings) } @@ -95,10 +86,32 @@ fn read_mappings(results_folder: &Path) -> Result> { /// Reconstruct mappings across forks because inherited perf events do not /// synthesize mappings that already existed when a child was forked. fn read_mappings_from_artifact(reader: R) -> Result> { - let stream = MemtrackArtifact::decode_streamed(reader)?; - let mut timeline = Vec::new(); + let mut timeline = MemtrackArtifact::decode_streamed(reader)? + .filter(|event| { + matches!( + &event.kind, + MemtrackEventKind::Exec + | MemtrackEventKind::Mapping { .. } + | MemtrackEventKind::Fork { .. } + ) + }) + .collect::>(); + + // Ties break so exec purges before mapping, while fork inherits that mapping. + timeline.sort_by_key(|event| { + let rank = match &event.kind { + MemtrackEventKind::Exec => 0, + MemtrackEventKind::Mapping { .. } => 1, + MemtrackEventKind::Fork { .. } => 2, + _ => unreachable!(), + }; + (event.timestamp, rank) + }); + + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); - for event in stream { + for event in timeline { match event.kind { MemtrackEventKind::Mapping { path, @@ -112,7 +125,7 @@ fn read_mappings_from_artifact(reader: R) -> Result(reader: R) -> Result { - timeline.push(MappingTimelineEvent::Fork { - parent_pid, - child_pid: event.pid, - timestamp: event.timestamp, - }); - } - MemtrackEventKind::Exec => { - timeline.push(MappingTimelineEvent::Exec { - pid: event.pid, - timestamp: event.timestamp, - }); - } - _ => {} - } - } - - timeline.sort_by_key(|event| event.order()); - - let mut live_mappings: HashMap> = HashMap::new(); - let mut mappings = Vec::new(); - - for event in timeline { - match event { - MappingTimelineEvent::Mapping(mapping) => { + }; live_mappings .entry(mapping.pid) .or_default() .push(mapping.clone()); mappings.push(mapping); } - MappingTimelineEvent::Fork { - parent_pid, - child_pid, - timestamp, - } => { + MemtrackEventKind::Fork { parent_pid } => { let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); let child_mappings = inherited .into_iter() .map(|mut mapping| { - mapping.pid = child_pid; - mapping.timestamp = timestamp; + mapping.pid = event.pid; + mapping.timestamp = event.timestamp; mapping }) .collect::>(); mappings.extend(child_mappings.iter().cloned()); - live_mappings.insert(child_pid, child_mappings); + live_mappings.insert(event.pid, child_mappings); } - MappingTimelineEvent::Exec { pid, .. } => { - live_mappings.remove(&pid); + MemtrackEventKind::Exec => { + live_mappings.remove(&event.pid); } + _ => unreachable!(), } } @@ -210,22 +194,29 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { - process_unwind_data.timestamp = Some(mapping.timestamp); - Some((unwind_data, process_unwind_data)) - } - Err(e) => { - debug!("Failed to load unwind data for {}: {e}", mapping.path); - None + let process_unwind_data = if let Some(unwind_data) = &loaded_module.unwind_data { + Some(ProcessUnwindData { + timestamp: Some(mapping.timestamp), + avma_range: mapping.avma_range.clone(), + base_avma: unwind_data.base_svma.wrapping_add(load_bias), + }) + } else { + match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + loaded_module.unwind_data = Some(unwind_data); + process_unwind_data.timestamp = Some(mapping.timestamp); + Some(process_unwind_data) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } } }; @@ -235,8 +226,7 @@ fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 24/27] fixup! refactor(runner): move ELF artifact pipeline to executor/shared --- src/executor/helpers/debug_file.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index e3bbd506..0619bed2 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,12 +12,6 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { - if let Some(dir) = binary_path.parent() { - if let Some(path) = find_debug_file_in(object, binary_path, dir) { - return Some(path); - } - } - ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) From b8b3e03d9fc9d5cf6736a730b575a333baaba079 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 25/27] fixup! feat(memtrack): add userspace stack-capture module --- crates/memtrack/src/ebpf/events.rs | 143 +++++++++++++++++++++- crates/memtrack/src/ebpf/stacks/config.rs | 6 - crates/memtrack/src/ebpf/stacks/events.rs | 143 ---------------------- crates/memtrack/src/ebpf/stacks/mod.rs | 2 - 4 files changed, 142 insertions(+), 152 deletions(-) delete mode 100644 crates/memtrack/src/ebpf/stacks/config.rs delete mode 100644 crates/memtrack/src/ebpf/stacks/events.rs diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 79597357..5a8abfce 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -1,4 +1,6 @@ -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; // Include the bindings for event.h pub mod bindings { @@ -120,6 +122,74 @@ pub fn parse_event(data: &[u8]) -> Option { }) } +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] pub struct AttachRequest { @@ -294,3 +364,74 @@ mod tests { } } } + +#[cfg(test)] +mod stack_tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs deleted file mode 100644 index 396817ad..00000000 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub fn stack_capture_from_env() -> bool { - !matches!( - std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), - Ok("0") | Ok("false") - ) -} diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs deleted file mode 100644 index 5d1b6fbf..00000000 --- a/crates/memtrack/src/ebpf/stacks/events.rs +++ /dev/null @@ -1,143 +0,0 @@ -use crate::ebpf::events::bindings::stack_header; -use crate::prelude::*; -use libbpf_rs::MapCore; -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; - -/// Decode one stack record from the ring buffer, returning it alongside the -/// `bpf_get_stackid()` result its frame-pointer chain is stored under. -pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { - let header_len = std::mem::size_of::(); - // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. - let header: stack_header = if data.len() >= header_len { - unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } - } else { - warn!( - "malformed stack record: {} bytes, need at least {header_len}", - data.len() - ); - return None; - }; - - let record_len = header_len + header.copy_len as usize; - if data.len() < record_len { - warn!( - "malformed stack record: {} bytes, need {record_len}", - data.len() - ); - return None; - } - - let event = MemtrackEvent { - pid: header.pid as i32, - tid: header.tid as i32, - timestamp: header.timestamp, - addr: 0, - kind: MemtrackEventKind::Stack { - record: Box::new(StackRecord { - hash: header.hash, - sp: header.sp, - regs: header.regs.reg.to_vec(), - bytes: data[header_len..record_len].to_vec(), - fp_chain: Vec::new(), - truncated: header.truncated != 0, - }), - }, - }; - - Some((event, header.stackid)) -} - -/// The frame-pointer walk recorded under `stackid`, innermost frame first. -/// Best effort: a missing chain costs the fallback for one stack, not the run. -pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { - let Ok(key) = u32::try_from(stackid) else { - return Vec::new(); - }; - - let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { - Ok(Some(value)) => value, - Ok(None) => return Vec::new(), - Err(error) => { - warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); - return Vec::new(); - } - }; - - // The map value is a fixed-depth array zero-padded past the last frame. - value - .chunks_exact(8) - .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) - .take_while(|&address| address != 0) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::ebpf::events::bindings::stack_regs; - - fn encode(header: stack_header, payload: &[u8]) -> Vec { - // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. - let header_bytes = unsafe { - std::slice::from_raw_parts( - (&header as *const stack_header).cast::(), - std::mem::size_of::(), - ) - }; - let mut data = header_bytes.to_vec(); - data.extend_from_slice(payload); - data - } - - fn header(copy_len: u32) -> stack_header { - stack_header { - hash: 0x0123_4567_89ab_cdef, - timestamp: 987_654_321, - stackid: -17, - sp: 0x7fff_1234_5000, - pid: 41, - tid: 42, - copy_len, - truncated: 1, - _pad: [0; 3], - regs: stack_regs { - reg: std::array::from_fn(|index| 0x1000 + index as u64), - }, - } - } - - #[test] - fn well_formed_record_round_trips_every_field() { - let header = header(5); - let payload = [1, 2, 3, 4, 5]; - - let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); - assert_eq!(event.pid, 41); - assert_eq!(event.tid, 42); - assert_eq!(event.timestamp, 987_654_321); - assert_eq!(event.addr, 0); - assert_eq!(stackid, -17); - - let MemtrackEventKind::Stack { record } = event.kind else { - panic!("expected Stack event"); - }; - - assert_eq!(record.hash, header.hash); - assert_eq!(record.sp, header.sp); - assert_eq!(record.regs, header.regs.reg.to_vec()); - assert_eq!(record.bytes, payload); - assert!(record.fp_chain.is_empty()); - assert!(record.truncated); - } - - #[test] - fn truncated_buffer_returns_none() { - let data = vec![0; std::mem::size_of::() - 1]; - assert!(parse_stack(&data).is_none()); - } - - #[test] - fn missing_payload_returns_none() { - assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); - } -} diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs index fc5eee33..25b17c6e 100644 --- a/crates/memtrack/src/ebpf/stacks/mod.rs +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -1,3 +1 @@ -pub mod config; pub mod counters; -pub mod events; From b7beba06043f736537036a22752c6c30e43700a6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 26/27] fixup! feat(memtrack): enable stack capture through the tracker --- crates/memtrack/src/ebpf/memtrack/mod.rs | 33 +++++-------- crates/memtrack/src/ebpf/tracker.rs | 63 ++++++++++-------------- crates/memtrack/src/main.rs | 7 +++ 3 files changed, 47 insertions(+), 56 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 0644b176..dc4e8274 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -7,6 +7,7 @@ use std::mem::MaybeUninit; use std::path::Path; use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::tracker::TrackerOptions; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -125,25 +126,17 @@ pub struct MemtrackBpf { } impl MemtrackBpf { - /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool, capture_stacks: bool) -> Result { - let variant = if has_delegated_bpf_token() { - BpfVariant::Token - } else { - BpfVariant::Legacy - }; - Self::with_variant(variant, track_rmap, capture_stacks) - } - - /// Load a specific variant rather than the one [`Self::new_with_rmap`] - /// would detect. Either attaches given host privileges; the token only - /// matters when `bpf()` is called from an unprivileged user namespace. - /// `capture_stacks` enables allocation stack capture. - pub fn with_variant( - variant: BpfVariant, - track_rmap: bool, - capture_stacks: bool, - ) -> Result { + /// Load the skeleton using the requested configuration. + pub fn new(options: &TrackerOptions) -> Result { + let variant = options.variant.unwrap_or_else(|| { + if has_delegated_bpf_token() { + BpfVariant::Token + } else { + BpfVariant::Legacy + } + }); + let track_rmap = options.rmap; + let capture_stacks = options.stack_capture; let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -253,7 +246,7 @@ impl MemtrackBpf { poll_interval_ms: u64, tx: std::sync::mpsc::Sender, ) -> Result { - use crate::ebpf::stacks::events; + use crate::ebpf::events; use runner_shared::artifacts::MemtrackEventKind; // The poller outlives this borrow of the skeleton, so the chain lookup diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index b44afb44..7f8397fa 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,6 +1,5 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::stack_capture_from_env; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::perf_mappings::PerfMappingPoller; @@ -10,12 +9,15 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; #[derive(Debug, Clone, Copy, TypedBuilder)] pub struct TrackerOptions { + /// BPF attach mechanism, or automatic detection when unset. + #[builder(default)] + pub variant: Option, /// Attach allocator uprobes (malloc/free/calloc/...) through the /// exec-mapping watcher. #[builder(default = true)] @@ -36,18 +38,24 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) - .stack_capture(stack_capture_from_env()) + .stack_capture(!matches!( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), + Ok("0") | Ok("false") + )) .build() } } +impl Default for TrackerOptions { + fn default() -> Self { + Self::builder().build() + } +} + pub struct Tracker { bpf: Arc>, worker: Mutex>, - allocators: bool, - /// The dedup gate spans the whole BPF object, so a second session would - /// reference stack records the first one already consumed. - stacks_polled: Option, + options: TrackerOptions, /// Number of native perf mapping records lost due to ring-buffer overflow. mapping_lost: Arc, } @@ -67,37 +75,23 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - let capture_stacks = options.stack_capture; - Self::build( - MemtrackBpf::new_with_rmap(options.rmap, capture_stacks)?, - options.allocators, - capture_stacks, - ) - } - /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of - /// the detected one. - pub fn with_variant(variant: BpfVariant) -> Result { - let track_rmap = TrackerOptions::from_env().rmap; - Self::build( - MemtrackBpf::with_variant(variant, track_rmap, false)?, - true, - false, - ) + let bpf = MemtrackBpf::new(&options)?; + Self::build(bpf, options) } /// Build a tracker: attach lifetime tracepoints (and rmap fentries when the /// skeleton was opened for them), plus, when `allocators` is set, the /// exec-mapping watcher and the on-demand allocator-attach worker. - fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { + fn build(mut bpf: MemtrackBpf, options: TrackerOptions) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; - if allocators { + if options.allocators { bpf.attach_exec_watcher()?; } let bpf = Arc::new(Mutex::new(bpf)); - let worker = if allocators { + let worker = if options.allocators { Some(AttachWorker::start(bpf.clone())?) } else { None @@ -106,8 +100,7 @@ impl Tracker { Ok(Self { bpf, worker: Mutex::new(worker), - allocators, - stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), + options, mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -121,13 +114,7 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { - let capture_stacks = match &self.stacks_polled { - Some(polled) if polled.swap(true, Ordering::Relaxed) => { - bail!("stack capture supports a single spawned command per tracker") - } - Some(_) => true, - None => false, - }; + let capture_stacks = self.options.stack_capture; let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { @@ -141,7 +128,7 @@ impl Tracker { match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), + None if self.options.allocators => bail!("tracker already finished"), None => {} } @@ -204,6 +191,10 @@ impl Tracker { self.bpf.lock().stack_capture_stats() } + pub fn stack_capture_enabled(&self) -> bool { + self.options.stack_capture + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff19..d9a6411b 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,13 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + if tracker.stack_capture_enabled() { + let stats = tracker + .stack_capture_stats() + .context("Failed to read stack capture stats")?; + debug!("stack capture stats: {stats:?}"); + } + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. From 84c37d003d06875f2bc0b8f2c1b6d9291067f458 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Wed, 2 Sep 2026 11:29:08 +0200 Subject: [PATCH 27/27] fixup! test(memtrack): cover allocation stack capture --- crates/memtrack/tests/c_tests.rs | 2 +- crates/memtrack/tests/rss_tests.rs | 24 +- crates/memtrack/tests/shared.rs | 36 +- .../stack_tests__nested_doubling.snap | 12 + ...ck_tests__nested_doubling_shared_free.snap | 12 + .../stack_tests__stack_capture_disabled.snap | 206 +++++++++++ .../snapshots/stack_tests__stack_paths.snap | 206 +++++++++++ crates/memtrack/tests/stack_tests.rs | 331 +++++------------- 8 files changed, 550 insertions(+), 279 deletions(-) create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap create mode 100644 crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index 62b13028..ab83ae80 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -99,7 +99,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box TrackerOptions { + TrackerOptions::builder() + .allocators(false) + .rmap(true) + .build() +} + /// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to /// its argv[1] alongside the collected events. /// @@ -349,7 +357,9 @@ fn test_rss_rmap_tracking( #[case] source: &str, #[case] name: &str, ) -> Result<(), Box> { - let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (raw_report, events) = track_fixture(source, name, |command| { + shared::track_command(command, rmap_only_options()) + })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); let summary = RssSummary { @@ -425,14 +435,14 @@ enum Reclaim { #[case::rss_stat(Reclaim::RssStat)] #[case::rmap(Reclaim::Rmap)] fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { - let track: fn(Command) -> shared::TrackResult = match mode { - Reclaim::RssStat => shared::track_command, - Reclaim::Rmap => shared::track_command_with_rmap, + let options = match mode { + Reclaim::RssStat => TrackerOptions::default(), + Reclaim::Rmap => rmap_only_options(), }; let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - track, + |command| shared::track_command(command, options), )?; // A = owner that faulted the file region; B = external caller, single-threaded @@ -601,7 +611,7 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Result<(), Box TrackResult { - track_command(Command::new(binary)) + track_command(Command::new(binary), None) } pub fn track_binary_with_env(binary: &Path) -> TrackResult { @@ -205,15 +205,10 @@ pub fn compile_c_source( Ok(binary_path) } -/// Track a command with the default probes: no rmap, and allocators discovered -/// by the exec-mapping watcher as the tracked tree maps executables. -pub fn track_command(command: Command) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().build()) -} - -/// Track a command under a specific BPF variant rather than the detected one. -pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> TrackResult { - track_command_with_tracker(command, Tracker::with_variant(variant)?) +/// Track a command with explicit options, using builder defaults when absent. +pub fn track_command(command: Command, options: impl Into>) -> TrackResult { + let tracker = Tracker::with_options(options.into().unwrap_or_default())?; + track_command_with_tracker(command, tracker) } /// RSS reconstruction from the folio rmap hooks, without allocator probes. @@ -224,16 +219,6 @@ fn rmap_only_options() -> TrackerOptions { .build() } -/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. -pub fn track_command_with_rmap(command: Command) -> TrackResult { - track_command_with_opts(command, rmap_only_options()) -} - -/// Track a command with an explicit probe selection rather than the environment's. -pub fn track_command_with_opts(command: Command, options: TrackerOptions) -> TrackResult { - track_command_with_tracker(command, Tracker::with_options(options)?) -} - /// Track a command with rmap hooks and snapshot its ownership maps after the /// tracked tree exits but before tracker teardown frees the BPF maps. pub fn track_command_with_rmap_maps( @@ -245,14 +230,6 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } -/// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks(command: Command) -> TrackResult { - track_command_with_opts( - command, - TrackerOptions::builder().stack_capture(true).build(), - ) -} - /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// @@ -323,7 +300,8 @@ pub fn for_each_variant( let mut profiles: Vec<(BpfVariant, EventProfile)> = Vec::new(); for variant in [BpfVariant::Legacy, BpfVariant::Token] { - let tracker = match Tracker::with_variant(variant) { + let options = TrackerOptions::builder().variant(Some(variant)).build(); + let tracker = match Tracker::with_options(options) { Ok(tracker) => tracker, Err(err) => { eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap new file mode 100644 index 00000000..f1d65a99 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap new file mode 100644 index 00000000..f1d65a99 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap new file mode 100644 index 00000000..cf9fc935 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap new file mode 100644 index 00000000..7cb31d31 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 5dffa3a9..0a0c7b25 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -1,267 +1,114 @@ #[macro_use] mod shared; +use itertools::Itertools; +use memtrack::TrackerOptions; +use rstest::rstest; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; -use std::collections::HashSet; +use std::mem::discriminant; use std::process::Command; use tempfile::TempDir; -const COPY_SIZE: u32 = 8192; - -fn compile_fixture( - name: &str, - temp_dir: &TempDir, -) -> Result> { - shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - name, - temp_dir.path(), - ) -} - -/// The stack identity carried by each allocation and deallocation event that has one. -fn event_hashes(events: &[MemtrackEvent]) -> Vec { - events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } - | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), - _ => None, - }) - .collect() -} +fn describe_allocator_event(kind: &MemtrackEventKind) -> Option { + let description = match kind { + MemtrackEventKind::Malloc { size, stack_hash } => { + format!("Malloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::Calloc { size, stack_hash } => { + format!("Calloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::AlignedAlloc { size, stack_hash } => format!( + "AlignedAlloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ), + MemtrackEventKind::Realloc { + size, stack_hash, .. + } => { + format!( + "Realloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ) + } + MemtrackEventKind::Free { stack_hash } => { + format!("Free {{ has_stack: {} }}", *stack_hash != 0) + } + _ => return None, + }; -fn record_hashes(events: &[MemtrackEvent]) -> HashSet { - events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record } => Some(record.hash), - _ => None, - }) - .collect() + Some(description) } -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let records: Vec<_> = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::Stack { record: r } => { - Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) - } - _ => None, - }) - .collect(); - - assert!( - records.len() >= 2, - "expected at least two stack records, got {} ({} events)", - records.len(), - events.len() - ); - - let hashes = record_hashes(&events); - assert_eq!( - hashes.len(), - records.len(), - "stack records must be deduplicated by unique hash" - ); - - for (hash, sp, regs, bytes, truncated) in &records { - assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); - assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); - assert!( - !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, - "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", - bytes.len() - ); - assert_eq!( - *truncated, - bytes.len() == COPY_SIZE as usize, - "record {hash:#x} may only be flagged truncated when it filled the budget" - ); - } - - let carried = event_hashes(&events); - assert!( - !carried.is_empty(), - "expected events carrying a captured stack hash" - ); - assert!( - carried.iter().all(|hash| hashes.contains(hash)), - "every non-zero stack_hash must have a matching stack record" - ); - - // The fixture frees every allocation, so both sides must report identities. - assert!( +fn format_events(events: &[MemtrackEvent]) -> Vec { + const MARKER: u64 = 0xC0D5_9EED; + let has_markers = events.iter().any(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { size, .. } if size == MARKER + ) + }); + + let filtered_events = if has_markers { + shared::between_markers(events) + } else { events .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), - "free events must carry their own stack identity" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) -} - -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let carried = event_hashes(&events); - let records = record_hashes(&events); - assert!( - carried.len() > records.len(), - "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", - carried.len(), - records.len(), - events.len() - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - Ok(()) -} - -/// Restores the capture toggle on drop so a failing assertion cannot leak the -/// override into later tests (the suite runs single-threaded). -struct DisableCaptureGuard; - -impl DisableCaptureGuard { - fn set() -> Self { - // SAFETY: tests run with --test-threads 1, so no concurrent env access. - unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; - Self - } -} + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) + }) + .sorted_by_key(|e| e.timestamp) + .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) + .cloned() + .collect() + }; -impl Drop for DisableCaptureGuard { - fn drop(&mut self) { - // SAFETY: see `set`. - unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; - } + filtered_events + .iter() + .filter_map(|e| describe_allocator_event(&e.kind)) + .collect() } #[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::stack_paths(include_str!("../testdata/stack_paths.c"), "stack_paths", true)] +#[case::nested_doubling( + include_str!("../testdata/nested_doubling.c"), + "nested_doubling", + true +)] +#[case::nested_doubling_shared_free( + include_str!("../testdata/nested_doubling_shared_free.c"), + "nested_doubling_shared_free", + true +)] +#[case::stack_capture_disabled( + include_str!("../testdata/stack_paths.c"), + "stack_capture_disabled", + false +)] #[test_log::test] -fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { +fn test_stack_capture( + #[case] source: &str, + #[case] name: &str, + #[case] stack_capture: bool, +) -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; - let _guard = DisableCaptureGuard::set(); - let (events, thread_handle) = shared::track_binary_with_env(&binary)?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let options = TrackerOptions::builder() + .stack_capture(stack_capture) + .build(); + let (events, thread_handle) = shared::track_command(Command::new(binary), options)?; - assert!( - events - .iter() - .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), - "disabled capture must still report allocation events" - ); - assert!( - record_hashes(&events).is_empty(), - "disabled capture must emit zero stack records" - ); - assert!( - event_hashes(&events).is_empty(), - "disabled capture must leave stack_hash zero on every event" - ); + insta::assert_debug_snapshot!(name, format_events(&events)); thread_handle .join() .expect("tracker teardown thread panicked"); Ok(()) } - -/// The doubling fixtures allocate down a three-level call chain and free in the -/// reverse order, from three distinct depths (`nested_doubling.c`) or from the -/// innermost frame (`nested_doubling_shared_free.c`). Both must pair every free -/// with its allocation in reverse order and give each allocation depth its own -/// stack identity. -#[test_with::env(GITHUB_ACTIONS)] -#[test_log::test] -fn nested_doubling_frees_in_reverse_order() -> Result<(), Box> { - for (name, source) in [ - ( - "nested_doubling", - include_str!("../testdata/nested_doubling.c"), - ), - ( - "nested_doubling_shared_free", - include_str!("../testdata/nested_doubling_shared_free.c"), - ), - ] { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source(source, name, temp_dir.path())?; - let (events, thread_handle) = shared::track_command_with_stacks(Command::new(&binary))?; - - let allocations: Vec<(u64, u64, u64)> = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size, stack_hash } if (1024..=4096).contains(&size) => { - Some((size, e.addr, stack_hash)) - } - _ => None, - }) - .collect(); - - assert_eq!( - allocations - .iter() - .map(|(size, ..)| *size) - .collect::>(), - vec![1024, 2048, 4096], - "[{name}] each level must allocate twice its caller, outermost first" - ); - - // Both probes of one free report the same address, so dedup by address - // to recover the order the fixture released its buffers in. - let mut released: Vec = Vec::new(); - let mut free_hashes: Vec = Vec::new(); - for event in &events { - let MemtrackEventKind::Free { stack_hash } = event.kind else { - continue; - }; - if released.last() == Some(&event.addr) { - continue; - } - released.push(event.addr); - free_hashes.push(stack_hash); - } - - let allocated: Vec = allocations.iter().map(|(_, addr, _)| *addr).collect(); - let expected: Vec = allocated.iter().rev().copied().collect(); - assert_eq!( - released, expected, - "[{name}] buffers must be freed in the reverse of their allocation order" - ); - - let alloc_hashes: HashSet = allocations.iter().map(|(.., hash)| *hash).collect(); - assert_eq!( - alloc_hashes.len(), - allocations.len(), - "[{name}] each allocation depth must carry its own stack identity" - ); - assert!( - !alloc_hashes.contains(&0) && !free_hashes.contains(&0), - "[{name}] every allocation and free must carry a captured stack" - ); - - thread_handle - .join() - .expect("tracker teardown thread panicked"); - } - Ok(()) -}