diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75eac..3265a9194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/Cargo.lock b/Cargo.lock index 1acec08f4..4956c2d3b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2402,6 +2402,7 @@ dependencies = [ "object", "parking_lot", "paste", + "perf-event-open-sys", "rayon", "rstest", "runner-shared", @@ -2813,6 +2814,15 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "perf-event-open-sys" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5f8d1487a4ffa23c80a1c355dd27235f9b66fb71ba0f261eb417e4fe8451347" +dependencies = [ + "libc", +] + [[package]] name = "pest" version = "2.8.6" @@ -3616,6 +3626,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4062,6 +4073,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/Cargo.toml b/Cargo.toml index 8bd28f039..34be934d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -111,6 +111,7 @@ ipc-channel = "0.20" itertools = "0.14.0" rayon = "1.12" linux-perf-event-reader = "0.10.2" # matches the version linux-perf-data resolves to +perf-event-open-sys = "6.0" env_logger = "0.11.10" tempfile = "3.27.0" object = { version = "0.39", default-features = false, features = ["read_core", "elf"] } diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d41..2439bf0f0 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,20 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping collection (`src/perf_mappings.rs`) uses Linux's native per-CPU perf event stream, not an LSM/BPF availability gate. `PerfMappingPoller` opens a `PERF_TYPE_SOFTWARE` dummy event with `PERF_ATTR_INHERIT` and `PERF_ATTR_MMAP2` on every online CPU for the tracked process, mmaps a perf ring per CPU, and drains those rings on a poll thread. It keeps executable mappings with absolute paths from `PERF_RECORD_MMAP2` and emits the single artifact representation, `MemtrackEventKind::Mapping` inside `MemtrackArtifact.events`, carrying the mapping's pid/tid/timestamp/address/path/device/inode/file offset/length. Opening or enabling any perf event requires the host's perf permissions (for example an allowed `perf_event_paranoid` policy or `CAP_PERFMON`); a permission error is returned from `Tracker::spawn` rather than silently disabling mapping collection. Kernel `PERF_RECORD_LOST` records, ring overruns, and malformed records increment the shared mapping-loss counter. `Tracker::dropped_events_count()` includes that counter with BPF ring-buffer drops, and `codspeed-memtrack track` aborts when the total is non-zero because the artifact is incomplete.` + +### Event stream compatibility + +Session relies on Rust's declaration-order field drop: _poller, _stack_poller, then _perf_mapping_poller. The BPF event and stack pollers therefore disconnect, fully drain, and join before the perf poller is dropped. PerfMappingPoller buffers mapping records and emits them during shutdown, after ordinary allocation/RSS/stack events have reached encode_events; encode_events preserves input order, so Mapping records are a terminal suffix in the one artifact stream. + +This ordering is compatibility-critical. Mapping is a newer event variant; older stream consumers may treat the first unknown Mapping as EOF. Keeping it as the suffix lets those consumers process the complete memory timeline before stopping at that first unknown record. Do not reorder the poller fields or emit mapping records before shutdown. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `stacks/`, `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/stack_capture.bpf.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/perf_mappings.rs` — native per-CPU `PERF_RECORD_MMAP2` collector. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index 4912802da..68da1218f 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -34,6 +34,7 @@ itertools = { workspace = true } paste = "1.0.15" libbpf-rs = { version = "0.26", features = ["vendored"], optional = true } object = { workspace = true } +perf-event-open-sys = { workspace = true } rayon = "1.12" parking_lot = "0.12" typed-builder = "0.23.2" diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc2387..8de96317f 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index e188c7d5f..90cbe4360 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,11 +14,6 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 -struct inode_key { - __u64 dev; - __u64 ino; -}; - /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c93..e413007b8 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,48 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. The scratch buffer holding + * header plus bytes is a per-CPU map value, capped at PCPU_MIN_UNIT_SIZE + * (32 KiB) by the kernel allocator. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024 - 512) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_PREEMPTED 5 +#define MEMTRACK_STACK_COUNTER_COUNT 6 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +71,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ @@ -69,6 +114,13 @@ struct event { } data; }; +/* Identifies a mapped file for the exec-mapping watcher. `dev` uses the + * kernel's s_dev encoding: (major << 20) | minor. */ +struct inode_key { + uint64_t dev; + uint64_t ino; +}; + /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff07..b405f572b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 000000000..3ac1b1cb4 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,242 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* Raw user-stack bytes and registers are emitted once per hash for offline + * DWARF unwinding. Allocation events carry the hash; bpf_get_stackid() supplies + * the frame-pointer fallback. + * + * Stack data changes may give one call path multiple hashes. + */ + +const volatile __u8 capture_stacks_enabled = 0; + +#define STACK_TRACE_MAX_DEPTH 127 +/* Captured lengths are rounded down to this granularity. */ +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +/* Frame-pointer fallback keyed by bpf_get_stackid(). */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* A separate ring keeps allocation events fixed-size. */ +BPF_RINGBUF(stacks, 64 * 1024 * 1024); +BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +/* The union permits word-wise hashing before emitting a variable-length record. */ +struct stack_scratch_buf { + struct stack_header header; + union { + __u8 bytes[MEMTRACK_MAX_STACK_COPY]; + __u64 words[MEMTRACK_MAX_STACK_COPY / 8]; + }; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct stack_scratch_buf); +} stack_scratch SEC(".maps"); + +/* Older clang BPF backends cannot lower a 32-bit atomic compare-and-swap. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} stack_busy SEC(".maps"); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + __u32 zero = 0; + struct stack_scratch_buf* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); + if (!scratch) { + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + const __u32 want = 8192; + + /* Chunked reads stop at the first unreadable stack region. Fixed-size, + * unrolled hash loops bound verifier state growth. */ + __u64 hash = FNV64_OFFSET; + __u32 got = 0; +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= MEMTRACK_MAX_STACK_COPY; + off += STACK_COPY_CHUNK) { + if (off >= want) { + break; + } + if (bpf_probe_read_user(&scratch->bytes[off], STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + __u32 base = off >> 3; +#pragma unroll + for (__u32 word = 0; word < STACK_COPY_CHUNK / 8; word++) { + hash = (hash ^ scratch->words[base + word]) * FNV64_PRIME; + } + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= want; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + /* Length distinguishes a full copy from the same bytes as a truncated prefix. + * Zero is reserved for allocation events without a stack. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST */ + return hash; + } + if (gate_result != 0) { + /* Re-emit when deduplication is full so the hash remains resolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + scratch->header.hash = hash; + scratch->header.timestamp = bpf_ktime_get_ns(); + scratch->header.stackid = stackid; + scratch->header.sp = sp; + scratch->header.pid = ids.tgid; + scratch->header.tid = ids.tid; + scratch->header.copy_len = got; + scratch->header.truncated = truncated; + scratch->header._pad[0] = 0; + scratch->header._pad[1] = 0; + scratch->header._pad[2] = 0; + fill_stack_regs(&scratch->header.regs, ctx); + + if (bpf_ringbuf_output(&stacks, scratch, sizeof(struct stack_header) + got, 0) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + bpf_map_delete_elem(&seen_stack_hashes, &hash); + } + + return hash; +} + +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + __u32 zero = 0; + __u64* busy = bpf_map_lookup_elem(&stack_busy, &zero); + if (!busy) { + return 0; + } + /* uprobe_multi lacks the bpf_prog_active recursion guard; preemption can + * re-enter on the same per-CPU scratch buffer. */ + if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); + return 0; + } + + __u64 hash = capture_stack_inner(ctx, ids); + + /* A same-CPU contender runs only after a context switch, which orders this + * plain store before scratch-buffer reuse. */ + *busy = 0; + return hash; +} + +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9a..ca53593d2 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a58..5a8abfce0 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -1,4 +1,6 @@ -use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; // Include the bindings for event.h pub mod bindings { @@ -34,13 +36,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +57,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -111,6 +122,74 @@ pub fn parse_event(data: &[u8]) -> Option { }) } +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] pub struct AttachRequest { @@ -157,6 +236,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +248,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +271,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +283,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } @@ -277,3 +364,74 @@ mod tests { } } } + +#[cfg(test)] +mod stack_tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d463..0a60f2705 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -68,6 +69,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 8586872e6..dc4e8274b 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -7,6 +7,7 @@ use std::mem::MaybeUninit; use std::path::Path; use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::tracker::TrackerOptions; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -20,11 +21,14 @@ mod macros; mod allocator; mod maps; mod rmap; +mod teardown; mod tracking; pub use maps::OwnershipMaps; pub use rmap::RmapSupport; +use teardown::FdHolder; + use crate::bpf_token::has_delegated_bpf_token; /// Which attach mechanism a loaded skeleton uses for its uprobes. See @@ -122,20 +126,17 @@ pub struct MemtrackBpf { } impl MemtrackBpf { - /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool) -> Result { - let variant = if has_delegated_bpf_token() { - BpfVariant::Token - } else { - BpfVariant::Legacy - }; - Self::with_variant(variant, track_rmap) - } - - /// Load a specific variant rather than the one [`Self::new_with_rmap`] - /// would detect. Either attaches given host privileges; the token only - /// matters when `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + /// Load the skeleton using the requested configuration. + pub fn new(options: &TrackerOptions) -> Result { + let variant = options.variant.unwrap_or_else(|| { + if has_delegated_bpf_token() { + BpfVariant::Token + } else { + BpfVariant::Legacy + } + }); + let track_rmap = options.rmap; + let capture_stacks = options.stack_capture; let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -164,6 +165,18 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if capture_stacks { + rodata.capture_stacks_enabled = 1; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if !capture_stacks { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so fentries whose targets @@ -227,6 +240,38 @@ impl MemtrackBpf { )) } + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + Some(event) + }; + + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stacks, + parse, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( @@ -247,28 +292,36 @@ impl MemtrackBpf { self.probes.len() } - /// Detach all BPF links in parallel. Closing a uprobe link blocks on two - /// RCU grace periods in the kernel, but concurrent waiters share grace - /// periods, so closing from many threads scales near-linearly. + /// Detach all BPF links without waiting for the kernel to complete it. + /// + /// Each release waits for a tasks-trace RCU grace period, and that wait is + /// unbounded: a kernel that has stopped completing them (a BPF program that + /// oopsed leaves a reader that never exits) blocks the caller forever, with + /// no way out even by exiting, since exiting closes the same descriptors. A + /// holder process takes them over, so the releases happen off the critical + /// path and the run always finishes with the trace it has. + /// + /// The probes stay installed until the holder is done, so a process starting + /// right after this returns can still trap into them. pub fn detach_probes(&mut self) { - const DETACH_THREADS: usize = 32; - - let mut probes = std::mem::take(&mut self.probes); + let probes = std::mem::take(&mut self.probes); if probes.is_empty() { return; } - debug!("Detaching {} BPF links", probes.len()); - let start = std::time::Instant::now(); - let chunk_size = probes.len().div_ceil(DETACH_THREADS); - std::thread::scope(|scope| { - while !probes.is_empty() { - let split_at = probes.len().saturating_sub(chunk_size); - let chunk = probes.split_off(split_at); - scope.spawn(move || drop(chunk)); + let count = probes.len(); + let holder = FdHolder::fork() + .inspect_err(|e| warn!("Detaching without a descriptor holder: {e:#}")) + .ok(); + drop(probes); + + match holder { + Some(holder) => { + debug!("Releasing {count} BPF links in holder {}", holder.pid()); + holder.release(); } - }); - debug!("Detached BPF links in {:?}", start.elapsed()); + None => debug!("Detached {count} BPF links"), + } } } diff --git a/crates/memtrack/src/ebpf/memtrack/teardown.rs b/crates/memtrack/src/ebpf/memtrack/teardown.rs new file mode 100644 index 000000000..2214ba97a --- /dev/null +++ b/crates/memtrack/src/ebpf/memtrack/teardown.rs @@ -0,0 +1,117 @@ +use crate::prelude::*; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +/// A forked process that keeps a second reference to every descriptor this +/// process holds, so releasing them here costs nothing. +/// +/// Releasing a classic uprobe link runs `perf_event_detach_bpf_prog()`, which +/// waits for a tasks-trace RCU grace period. That wait is unbounded: a kernel +/// whose grace periods have stalled never completes it, and the process cannot +/// escape it by exiting either, since exiting closes the very same descriptors. +/// A descriptor closed while another process still holds it only drops a +/// refcount, so with a holder alive the release — and the wait — happens in the +/// holder instead, off the critical path of whoever forked it. +pub(super) struct FdHolder { + pid: libc::pid_t, + /// Closing this releases the holder from its wait. + release: OwnedFd, +} + +impl FdHolder { + /// Fork a holder for the descriptors currently open. Descriptors opened + /// afterwards are not covered. + pub(super) fn fork() -> Result { + let (read_end, write_end) = pipe()?; + + // SAFETY: the child touches nothing but the pipe and _exit(), so it + // cannot deadlock on a lock a thread of the parent held across the fork. + let pid = unsafe { libc::fork() }; + ensure!( + pid >= 0, + "fork() failed: {}", + std::io::Error::last_os_error() + ); + + if pid == 0 { + unsafe { hold_until_released(read_end.as_raw_fd(), write_end.as_raw_fd()) }; + } + + Ok(Self { + pid, + release: write_end, + }) + } + + /// Process id of the holder, for logging. + pub(super) fn pid(&self) -> libc::pid_t { + self.pid + } + + /// Let the holder release the descriptors. It is not waited for: it finishes + /// whenever the kernel lets it and is reaped by init. + pub(super) fn release(self) { + drop(self.release); + } +} + +fn pipe() -> Result<(OwnedFd, OwnedFd)> { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: pipe() writes two descriptors into `fds`. + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + ensure!( + rc == 0, + "pipe() failed: {}", + std::io::Error::last_os_error() + ); + // SAFETY: both descriptors are fresh and owned by this process. + unsafe { Ok((OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1]))) } +} + +/// Block until the write end is closed, then exit, which closes every inherited +/// descriptor and performs whatever release work the last close of each entails. +/// +/// # Safety +/// +/// Only for the child of a `fork()`: it never returns, and calls nothing that is +/// unsafe to call between `fork()` and `_exit()` in a multi-threaded process. +unsafe fn hold_until_released(read_end: libc::c_int, write_end: libc::c_int) -> ! { + unsafe { + libc::close(write_end); + + let mut byte = 0u8; + loop { + let rc = libc::read(read_end, (&raw mut byte).cast(), 1); + if rc >= 0 || *libc::__errno_location() != libc::EINTR { + break; + } + } + + libc::_exit(0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The holder must outlive its parent's closes and exit only once released: + /// exiting early would make the parent's own close the last one and put the + /// unbounded wait back on the caller. + #[test] + fn holder_exits_only_once_released() { + let holder = FdHolder::fork().unwrap(); + let pid = holder.pid(); + + std::thread::sleep(std::time::Duration::from_millis(50)); + let mut status = 0; + // SAFETY: waitpid() writes only into `status`. + let reaped = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(reaped, 0, "holder exited before being released"); + + holder.release(); + // SAFETY: waitpid() writes only into `status`. It holds no descriptor + // whose release can block, so it exits as soon as it is released. + let reaped = unsafe { libc::waitpid(pid, &mut status, 0) }; + assert_eq!(reaped, pid, "holder did not exit after being released"); + } +} diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index 2aa96549d..5b482ecf4 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,11 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::counters::StackCaptureStats; pub use tracker::{Tracker, TrackerOptions}; diff --git a/crates/memtrack/src/ebpf/stacks/counters.rs b/crates/memtrack/src/ebpf/stacks/counters.rs new file mode 100644 index 000000000..6e6d76b10 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/counters.rs @@ -0,0 +1,38 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, + /// Captures skipped because another BPF program used the per-CPU scratch. + pub preempted: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + preempted: slot(map, MEMTRACK_STACK_COUNTER_PREEMPTED)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs new file mode 100644 index 000000000..25b17c6ea --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -0,0 +1 @@ +pub mod counters; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index dc9bc13ef..7f8397fa3 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,17 +1,23 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::mpsc; use typed_builder::TypedBuilder; #[derive(Debug, Clone, Copy, TypedBuilder)] pub struct TrackerOptions { + /// BPF attach mechanism, or automatic detection when unset. + #[builder(default)] + pub variant: Option, /// Attach allocator uprobes (malloc/free/calloc/...) through the /// exec-mapping watcher. #[builder(default = true)] @@ -19,6 +25,9 @@ pub struct TrackerOptions { /// Reconstruct per-process RSS from the folio rmap fentry hooks. #[builder(default = false)] pub rmap: bool, + /// Capture allocation call stacks. + #[builder(default = true)] + pub stack_capture: bool, } impl TrackerOptions { @@ -29,14 +38,32 @@ impl TrackerOptions { Ok("0") | Ok("false") )) .rmap(std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1")) + .stack_capture(!matches!( + std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref(), + Ok("0") | Ok("false") + )) .build() } } +impl Default for TrackerOptions { + fn default() -> Self { + Self::builder().build() + } +} + pub struct Tracker { bpf: Arc>, worker: Mutex>, - allocators: bool, + options: TrackerOptions, + /// Number of native perf mapping records lost due to ring-buffer overflow. + mapping_lost: Arc, +} + +fn kill_and_wait(child: &mut std::process::Child) { + // Cleanup is best effort so the setup error remains the returned error. + let _ = child.kill(); + let _ = child.wait(); } impl Tracker { @@ -48,32 +75,23 @@ impl Tracker { /// Create a tracker from an explicit probe selection rather than the environment. pub fn with_options(options: TrackerOptions) -> Result { - Self::build( - MemtrackBpf::new_with_rmap(options.rmap)?, - options.allocators, - ) - } - - /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of - /// the detected one. - pub fn with_variant(variant: BpfVariant) -> Result { - let track_rmap = TrackerOptions::from_env().rmap; - Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) + let bpf = MemtrackBpf::new(&options)?; + Self::build(bpf, options) } /// Build a tracker: attach lifetime tracepoints (and rmap fentries when the /// skeleton was opened for them), plus, when `allocators` is set, the /// exec-mapping watcher and the on-demand allocator-attach worker. - fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { + fn build(mut bpf: MemtrackBpf, options: TrackerOptions) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; - if allocators { + if options.allocators { bpf.attach_exec_watcher()?; } let bpf = Arc::new(Mutex::new(bpf)); - let worker = if allocators { + let worker = if options.allocators { Some(AttachWorker::start(bpf.clone())?) } else { None @@ -82,7 +100,8 @@ impl Tracker { Ok(Self { bpf, worker: Mutex::new(worker), - allocators, + options, + mapping_lost: Arc::new(AtomicU64::new(0)), }) } @@ -95,31 +114,60 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = self.options.stack_capture; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); } - let child = spawn_stopped(&mut wrapped)?; + let mut child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - match self.worker.lock().as_ref() { - Some(worker) => worker.set_root_pid(pid), - // No watcher to arm means exec mappings would be missed. - None if self.allocators => bail!("tracker already finished"), - None => {} - } - let (tx, rx) = mpsc::channel(); - let poller = { - let mut bpf = self.bpf.lock(); - bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let setup = (|| -> Result<_> { + match self.worker.lock().as_ref() { + Some(worker) => worker.set_root_pid(pid), + // No watcher to arm means exec mappings would be missed. + None if self.options.allocators => bail!("tracker already finished"), + None => {} + } + + let (tx, rx) = mpsc::channel(); + let (poller, stack_poller) = { + let mut bpf = self.bpf.lock(); + bpf.add_tracked_pid(pid)?; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + (bpf.poll_events_with_channel(10, tx.clone())?, stack_poller) + }; + let perf_mapping_poller = capture_stacks + .then(|| PerfMappingPoller::start(pid, tx, self.mapping_lost.clone())) + .transpose()?; + + Ok((rx, poller, stack_poller, perf_mapping_poller)) + })(); + let (rx, poller, stack_poller, perf_mapping_poller) = match setup { + Ok(pollers) => pollers, + Err(error) => { + kill_and_wait(&mut child); + return Err(error); + } }; - resume(pid)?; - Ok(Session::new(child, rx, poller)) - } + if let Err(error) = resume(pid) { + kill_and_wait(&mut child); + return Err(error); + } + Ok(Session::new( + child, + rx, + poller, + stack_poller, + perf_mapping_poller, + )) + } /// Enable allocator-event tracking in the BPF program. Lifetime events /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids /// regardless of this toggle. @@ -135,7 +183,16 @@ impl Tracker { /// Number of events the kernel dropped because the ring buffer was full. /// A non-zero value means the resulting trace is incomplete. pub fn dropped_events_count(&self) -> Result { - self.bpf.lock().dropped_events_count() + Ok(self.bpf.lock().dropped_events_count()? + self.mapping_lost.load(Ordering::Relaxed)) + } + + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + + pub fn stack_capture_enabled(&self) -> bool { + self.options.stack_capture } /// Only meaningful while the BPF object is alive; teardown frees the maps. diff --git a/crates/memtrack/src/lib.rs b/crates/memtrack/src/lib.rs index 1d3278d92..d8cd5f405 100644 --- a/crates/memtrack/src/lib.rs +++ b/crates/memtrack/src/lib.rs @@ -4,6 +4,8 @@ mod bpf_token; mod ebpf; mod ipc; mod kernel; +#[cfg(feature = "ebpf")] +mod perf_mappings; pub mod prelude; #[cfg(feature = "ebpf")] mod session; diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff194..d9a6411b1 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,13 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + if tracker.stack_capture_enabled() { + let stats = tracker + .stack_capture_stats() + .context("Failed to read stack capture stats")?; + debug!("stack capture stats: {stats:?}"); + } + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/perf_mappings.rs b/crates/memtrack/src/perf_mappings.rs new file mode 100644 index 000000000..45350dbf4 --- /dev/null +++ b/crates/memtrack/src/perf_mappings.rs @@ -0,0 +1,437 @@ +use crate::prelude::*; +use perf_event_open_sys::bindings::{ + PERF_COUNT_SW_DUMMY, PERF_FLAG_FD_CLOEXEC, PERF_RECORD_LOST, PERF_RECORD_MMAP2, + PERF_SAMPLE_TID, PERF_SAMPLE_TIME, PERF_TYPE_SOFTWARE, perf_event_attr, perf_event_header, + perf_event_mmap_page, +}; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::io; +use std::mem::size_of; +use std::os::fd::RawFd; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::{self, RecvTimeoutError, Sender}; +use std::thread::JoinHandle; +use std::time::Duration; + +const DATA_PAGES: usize = 64; + +struct PerfRing { + fd: RawFd, + mapping: *mut u8, + mapping_len: usize, + data_offset: usize, + data_size: usize, + enabled: bool, +} + +// The mapping is exclusively consumed by the poll thread. +unsafe impl Send for PerfRing {} + +impl PerfRing { + fn open(pid: libc::pid_t, cpu: u32, page_size: usize) -> Result { + let mapping_len = page_size + .checked_mul(DATA_PAGES + 1) + .context("perf ring mapping size overflow")?; + ensure!( + mapping_len >= size_of::(), + "perf ring mapping is smaller than its metadata page" + ); + let mut attr = perf_event_attr { + type_: PERF_TYPE_SOFTWARE, + size: size_of::() as u32, + config: PERF_COUNT_SW_DUMMY as u64, + sample_type: (PERF_SAMPLE_TID | PERF_SAMPLE_TIME) as u64, + // PERF_FORMAT_LOST cannot account for inherited child events from this + // parent fd, so PERF_RECORD_LOST remains the complete loss signal. + read_format: 0, + clockid: libc::CLOCK_MONOTONIC, + ..Default::default() + }; + attr.__bindgen_anon_2.wakeup_events = 1; + attr.set_disabled(1); + attr.set_inherit(1); + attr.set_mmap(1); + attr.set_sample_id_all(1); + attr.set_mmap2(1); + attr.set_use_clockid(1); + + let fd = unsafe { + perf_event_open_sys::perf_event_open( + &mut attr, + pid, + cpu as _, + -1, + PERF_FLAG_FD_CLOEXEC as _, + ) + }; + if fd < 0 { + return Err(io::Error::last_os_error()) + .with_context(|| format!("perf_event_open failed for pid {pid} on CPU {cpu}")); + } + + let mapping = unsafe { + libc::mmap( + ptr::null_mut(), + mapping_len, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_SHARED, + fd, + 0, + ) + }; + if mapping == libc::MAP_FAILED { + let error = io::Error::last_os_error(); + unsafe { libc::close(fd) }; + return Err(error).context("failed to mmap perf mapping-event ring buffer"); + } + + let page = unsafe { &*(mapping.cast::()) }; + let data_offset = match usize::try_from(page.data_offset) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data offset"); + } + }; + let data_size = match usize::try_from(page.data_size) { + Ok(value) => value, + Err(_) => { + unsafe { + libc::munmap(mapping, mapping_len); + libc::close(fd); + } + bail!("kernel returned an invalid perf ring data size"); + } + }; + let ring = Self { + fd, + mapping: mapping.cast(), + mapping_len, + data_offset, + data_size, + enabled: false, + }; + + ensure!( + data_offset >= page_size && data_offset % page_size == 0, + "kernel returned an invalid perf ring data offset" + ); + ensure!( + data_size >= size_of::() + && data_size % page_size == 0 + && data_size.is_power_of_two(), + "kernel returned an invalid perf ring data size" + ); + let data_end = data_offset + .checked_add(data_size) + .context("perf ring data range overflow")?; + ensure!( + data_end <= mapping_len, + "kernel returned a perf ring outside the mapped area" + ); + + Ok(ring) + } + + fn enable(&mut self) -> Result<()> { + if unsafe { perf_event_open_sys::ioctls::ENABLE(self.fd, 0) } < 0 { + return Err(io::Error::last_os_error()).context("failed to enable perf mapping events"); + } + self.enabled = true; + Ok(()) + } + + fn drain(&mut self, mappings: &mut Vec, lost: &AtomicU64) { + let page = unsafe { &mut *(self.mapping.cast::()) }; + let head = unsafe { ptr::read_volatile(&page.data_head) }; + std::sync::atomic::fence(Ordering::Acquire); + let mut tail = unsafe { ptr::read_volatile(&page.data_tail) }; + let available = head.wrapping_sub(tail); + + // Once the producer has lapped the consumer, the beginning of the + // stream no longer has a record boundary. Skip the corrupt prefix and + // let the kernel's PERF_RECORD_LOST record account for normal overflow. + if available > self.data_size as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + } else { + while tail != head { + let available = head.wrapping_sub(tail); + if available < size_of::() as u64 { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let header = self.copy_from_ring(tail, size_of::()); + let size = u16::from_ne_bytes([header[6], header[7]]) as usize; + if !(size_of::()..=self.data_size).contains(&size) + || size as u64 > available + { + lost.fetch_add(1, Ordering::Relaxed); + tail = head; + break; + } + + let record = self.copy_from_ring(tail, size); + self.handle_record(&record, mappings, lost); + tail = tail.wrapping_add(size as u64); + } + } + + std::sync::atomic::fence(Ordering::Release); + unsafe { ptr::write_volatile(&mut page.data_tail, tail) }; + } + + fn copy_from_ring(&self, offset: u64, len: usize) -> Vec { + debug_assert!(len <= self.data_size); + let start = offset as usize & (self.data_size - 1); + let first_len = len.min(self.data_size - start); + let data = unsafe { self.mapping.add(self.data_offset) }; + let mut out = Vec::with_capacity(len); + unsafe { + out.extend_from_slice(std::slice::from_raw_parts(data.add(start), first_len)); + if first_len < len { + out.extend_from_slice(std::slice::from_raw_parts(data, len - first_len)); + } + } + out + } + + fn handle_record(&self, record: &[u8], mappings: &mut Vec, lost: &AtomicU64) { + match read_u32(record, 0) { + Some(PERF_RECORD_MMAP2) => { + if let Some(event) = parse_mmap2(record) { + mappings.push(event); + } + } + Some(PERF_RECORD_LOST) => match read_u64(record, 16) { + Some(count) => { + lost.fetch_add(count, Ordering::Relaxed); + } + None => { + lost.fetch_add(1, Ordering::Relaxed); + } + }, + _ => {} + } + } +} + +impl Drop for PerfRing { + fn drop(&mut self) { + unsafe { + if self.enabled { + let _ = perf_event_open_sys::ioctls::DISABLE(self.fd, 0); + } + libc::munmap(self.mapping.cast(), self.mapping_len); + libc::close(self.fd); + } + } +} + +pub(crate) struct PerfMappingPoller { + ctl: Option>>, + thread: Option>, +} + +impl PerfMappingPoller { + pub(crate) fn start( + pid: libc::pid_t, + tx: Sender, + lost: Arc, + ) -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "failed to read the system page size"); + + let cpus = online_cpus()?; + ensure!(!cpus.is_empty(), "no online CPUs reported by the kernel"); + let mut rings = Vec::with_capacity(cpus.len()); + for cpu in cpus { + rings.push(PerfRing::open(pid, cpu, page_size as usize)?); + } + for ring in &mut rings { + ring.enable()?; + } + + let (ctl, ctl_rx) = mpsc::channel::>(); + let thread = std::thread::spawn(move || { + let mut mappings = Vec::new(); + loop { + match ctl_rx.recv_timeout(Duration::from_millis(10)) { + Ok(ack) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + let _ = ack.send(()); + } + Err(RecvTimeoutError::Timeout) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + } + Err(RecvTimeoutError::Disconnected) => { + for ring in &mut rings { + ring.drain(&mut mappings, &lost); + } + mappings.sort_unstable_by_key(|event| (event.pid, event.timestamp)); + for mapping in mappings { + let _ = tx.send(mapping); + } + break; + } + } + } + }); + + Ok(Self { + ctl: Some(ctl), + thread: Some(thread), + }) + } +} + +impl Drop for PerfMappingPoller { + fn drop(&mut self) { + drop(self.ctl.take()); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +fn parse_mmap2(record: &[u8]) -> Option { + const FIXED_END: usize = 72; + const SAMPLE_ID_SIZE: usize = 16; + if record.len() < FIXED_END + SAMPLE_ID_SIZE + || read_u32(record, 0)? != PERF_RECORD_MMAP2 + || read_u16(record, 6)? as usize != record.len() + { + return None; + } + + let prot = read_u32(record, 64)?; + if prot & libc::PROT_EXEC as u32 == 0 { + return None; + } + + let path_end = record.len() - SAMPLE_ID_SIZE; + let path_bytes = &record[FIXED_END..path_end]; + let nul = path_bytes.iter().position(|byte| *byte == 0)?; + let path = std::str::from_utf8(&path_bytes[..nul]).ok()?; + if !path.starts_with('/') { + return None; + } + + let major = read_u32(record, 40)? as u64; + let minor = read_u32(record, 44)? as u64; + Some(MemtrackEvent { + pid: read_u32(record, 8)? as libc::pid_t, + tid: read_u32(record, 12)? as libc::pid_t, + timestamp: read_u64(record, record.len() - 8)?, + addr: read_u64(record, 16)?, + kind: MemtrackEventKind::Mapping { + path: path.to_owned(), + dev: (major << 20) | minor, + ino: read_u64(record, 48)?, + file_offset: read_u64(record, 32)?, + len: read_u64(record, 24)?, + }, + }) +} + +fn read_u16(bytes: &[u8], offset: usize) -> Option { + Some(u16::from_ne_bytes( + bytes.get(offset..offset + 2)?.try_into().ok()?, + )) +} + +fn read_u32(bytes: &[u8], offset: usize) -> Option { + Some(u32::from_ne_bytes( + bytes.get(offset..offset + 4)?.try_into().ok()?, + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> Option { + Some(u64::from_ne_bytes( + bytes.get(offset..offset + 8)?.try_into().ok()?, + )) +} + +fn online_cpus() -> Result> { + let spec = std::fs::read_to_string("/sys/devices/system/cpu/online") + .context("failed to read online CPUs")?; + parse_cpu_list(spec.trim()) +} + +fn parse_cpu_list(spec: &str) -> Result> { + let mut cpus = Vec::new(); + for part in spec.split(',') { + let part = part.trim(); + ensure!(!part.is_empty(), "invalid empty CPU range"); + let (start, end) = match part.split_once('-') { + Some((start, end)) => (start.parse::()?, end.parse::()?), + None => { + let cpu = part.parse::()?; + (cpu, cpu) + } + }; + ensure!(start <= end, "invalid CPU range {part}"); + cpus.extend(start..=end); + } + Ok(cpus) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_cpu_ranges() { + assert_eq!(parse_cpu_list("0-2,5,8-9").unwrap(), vec![0, 1, 2, 5, 8, 9]); + } + + #[test] + fn parses_executable_mmap2() { + let path = b"/tmp/module.so\0"; + let mut record = vec![0; 72 + path.len() + 16]; + record[0..4].copy_from_slice(&PERF_RECORD_MMAP2.to_ne_bytes()); + let size = record.len() as u16; + record[6..8].copy_from_slice(&size.to_ne_bytes()); + record[8..12].copy_from_slice(&7_u32.to_ne_bytes()); + record[12..16].copy_from_slice(&8_u32.to_ne_bytes()); + record[16..24].copy_from_slice(&0x4000_u64.to_ne_bytes()); + record[24..32].copy_from_slice(&0x2000_u64.to_ne_bytes()); + record[32..40].copy_from_slice(&0x1000_u64.to_ne_bytes()); + record[40..44].copy_from_slice(&1_u32.to_ne_bytes()); + record[44..48].copy_from_slice(&2_u32.to_ne_bytes()); + record[48..56].copy_from_slice(&42_u64.to_ne_bytes()); + record[64..68].copy_from_slice(&(libc::PROT_EXEC as u32).to_ne_bytes()); + record[72..72 + path.len()].copy_from_slice(path); + let timestamp = 99_u64; + let time_offset = record.len() - 8; + record[time_offset..].copy_from_slice(×tamp.to_ne_bytes()); + + assert_eq!( + parse_mmap2(&record), + Some(MemtrackEvent { + pid: 7, + tid: 8, + timestamp, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: "/tmp/module.so".into(), + dev: (1 << 20) | 2, + ino: 42, + file_offset: 0x1000, + len: 0x2000, + }, + }) + ); + } +} diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33feb..c8f7ef475 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -1,4 +1,5 @@ use crate::ebpf::poller::RingBufferPoller; +use crate::perf_mappings::PerfMappingPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; @@ -9,7 +10,15 @@ use std::sync::mpsc::Receiver; pub struct Session { child: Child, events: Option>, + + // Drop order is part of the artifact compatibility contract. Rust drops + // fields in declaration order: both BPF pollers must stay before the perf + // mapping poller. Their Drop implementations disconnect, fully drain, and + // join their poll threads before PerfMappingPoller drops and emits its + // buffered Mapping records as the terminal stream suffix. _poller: RingBufferPoller, + _stack_poller: Option, + _perf_mapping_poller: Option, } impl Session { @@ -17,11 +26,15 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, + perf_mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, + _perf_mapping_poller: perf_mapping_poller, } } diff --git a/crates/memtrack/testdata/nested_doubling.c b/crates/memtrack/testdata/nested_doubling.c new file mode 100644 index 000000000..e3a588e3f --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling.c @@ -0,0 +1,41 @@ +#include +#include + +/* + * Each level allocates twice as much as its caller, then frees on the way back + * up, so the free order is the reverse of the allocation order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(1024) <-- free(2048) <-- free(4096) + * + * Every malloc and every free sits at a distinct call depth, so the six events + * also carry six distinct allocation stacks. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); +} + +__attribute__((noinline)) static void level2(void) { + void* p = malloc(2048); + escaped_pointer = p; + level3(); + free(p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(); + free(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/nested_doubling_shared_free.c b/crates/memtrack/testdata/nested_doubling_shared_free.c new file mode 100644 index 000000000..66c53fd99 --- /dev/null +++ b/crates/memtrack/testdata/nested_doubling_shared_free.c @@ -0,0 +1,44 @@ +#include +#include + +/* + * Same doubling allocation chain as nested_doubling.c, but ownership is handed + * down and the innermost level frees all three buffers in reverse order: + * + * level1 malloc(1024) --> level2 malloc(2048) --> level3 malloc(4096) + * free(4096) + * free(2048) + * free(1024) + * + * The three mallocs come from three different call depths while all three frees + * share one, so a deallocation event must be attributed to the free site rather + * than to wherever its allocation happened. + */ + +static volatile void* escaped_pointer; + +__attribute__((noinline)) static void level3(void* outer, void* middle) { + void* p = malloc(4096); + escaped_pointer = p; + free(p); + free(middle); + free(outer); +} + +__attribute__((noinline)) static void level2(void* outer) { + void* p = malloc(2048); + escaped_pointer = p; + level3(outer, p); +} + +__attribute__((noinline)) static void level1(void) { + void* p = malloc(1024); + escaped_pointer = p; + level2(p); +} + +int main() { + sleep(1); + level1(); + return 0; +} diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 000000000..0cee7f437 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/c_tests.rs b/crates/memtrack/tests/c_tests.rs index db5fe6439..ab83ae806 100644 --- a/crates/memtrack/tests/c_tests.rs +++ b/crates/memtrack/tests/c_tests.rs @@ -51,6 +51,14 @@ const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ name: "posix_memalign_einval", source: include_str!("../testdata/posix_memalign_einval.c"), }, + AllocationTestCase { + name: "nested_doubling", + source: include_str!("../testdata/nested_doubling.c"), + }, + AllocationTestCase { + name: "nested_doubling_shared_free", + source: include_str!("../testdata/nested_doubling_shared_free.c"), + }, ]; #[test_with::env(GITHUB_ACTIONS)] @@ -65,6 +73,8 @@ const ALLOCATION_TEST_CASES: &[AllocationTestCase] = &[ #[case(&ALLOCATION_TEST_CASES[7])] #[case(&ALLOCATION_TEST_CASES[8])] #[case(&ALLOCATION_TEST_CASES[9])] +#[case(&ALLOCATION_TEST_CASES[10])] +#[case(&ALLOCATION_TEST_CASES[11])] #[test_log::test] fn test_allocation_tracking( #[case] test_case: &AllocationTestCase, @@ -89,7 +99,7 @@ fn test_track_allocators_disabled_skips_allocations() -> Result<(), Box Result<(), Box Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index 5e45c04f0..8e8f8916d 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -2,6 +2,7 @@ mod shared; use itertools::Itertools; +use memtrack::TrackerOptions; use rstest::rstest; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; use serde::Serialize; @@ -205,6 +206,13 @@ fn build_fixture( Ok((temp_dir, report_path, command)) } +fn rmap_only_options() -> TrackerOptions { + TrackerOptions::builder() + .allocators(false) + .rmap(true) + .build() +} + /// Run a fixture under `track` and return the raw `/proc` RSS report it wrote to /// its argv[1] alongside the collected events. /// @@ -349,7 +357,9 @@ fn test_rss_rmap_tracking( #[case] source: &str, #[case] name: &str, ) -> Result<(), Box> { - let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (raw_report, events) = track_fixture(source, name, |command| { + shared::track_command(command, rmap_only_options()) + })?; let raw_report = raw_report.ok_or("fixture wrote no rss report")?; let (rss_stat, rmap) = per_pid_peaks(&events); let summary = RssSummary { @@ -425,14 +435,14 @@ enum Reclaim { #[case::rss_stat(Reclaim::RssStat)] #[case::rmap(Reclaim::Rmap)] fn test_rss_external_reclaim(#[case] mode: Reclaim) -> Result<(), Box> { - let track: fn(Command) -> shared::TrackResult = match mode { - Reclaim::RssStat => shared::track_command, - Reclaim::Rmap => shared::track_command_with_rmap, + let options = match mode { + Reclaim::RssStat => TrackerOptions::default(), + Reclaim::Rmap => rmap_only_options(), }; let (_report, events) = track_fixture( include_str!("../testdata/rss/madvise_extern.c"), "madvise_extern", - track, + |command| shared::track_command(command, options), )?; // A = owner that faulted the file region; B = external caller, single-threaded @@ -601,7 +611,7 @@ fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box Result<(), Box String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -108,7 +111,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -124,6 +127,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -173,7 +177,11 @@ pub fn compile_rust_binary( /// Track a binary, collecting all memory events. pub fn track_binary(binary: &Path) -> TrackResult { - track_command(Command::new(binary)) + track_command(Command::new(binary), None) +} + +pub fn track_binary_with_env(binary: &Path) -> TrackResult { + track_command_with_tracker(Command::new(binary), Tracker::new()?) } pub fn compile_c_source( @@ -197,15 +205,10 @@ pub fn compile_c_source( Ok(binary_path) } -/// Track a command with the default probes: no rmap, and allocators discovered -/// by the exec-mapping watcher as the tracked tree maps executables. -pub fn track_command(command: Command) -> TrackResult { - track_command_with_opts(command, TrackerOptions::builder().build()) -} - -/// Track a command under a specific BPF variant rather than the detected one. -pub fn track_command_with_variant(command: Command, variant: BpfVariant) -> TrackResult { - track_command_with_tracker(command, Tracker::with_variant(variant)?) +/// Track a command with explicit options, using builder defaults when absent. +pub fn track_command(command: Command, options: impl Into>) -> TrackResult { + let tracker = Tracker::with_options(options.into().unwrap_or_default())?; + track_command_with_tracker(command, tracker) } /// RSS reconstruction from the folio rmap hooks, without allocator probes. @@ -216,16 +219,6 @@ fn rmap_only_options() -> TrackerOptions { .build() } -/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. -pub fn track_command_with_rmap(command: Command) -> TrackResult { - track_command_with_opts(command, rmap_only_options()) -} - -/// Track a command with an explicit probe selection rather than the environment's. -pub fn track_command_with_opts(command: Command, options: TrackerOptions) -> TrackResult { - track_command_with_tracker(command, Tracker::with_options(options)?) -} - /// Track a command with rmap hooks and snapshot its ownership maps after the /// tracked tree exits but before tracker teardown frees the BPF maps. pub fn track_command_with_rmap_maps( @@ -281,7 +274,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } @@ -307,7 +300,8 @@ pub fn for_each_variant( let mut profiles: Vec<(BpfVariant, EventProfile)> = Vec::new(); for variant in [BpfVariant::Legacy, BpfVariant::Token] { - let tracker = match Tracker::with_variant(variant) { + let options = TrackerOptions::builder().variant(Some(variant)).build(); + let tracker = match Tracker::with_options(options) { Ok(tracker) => tracker, Err(err) => { eprintln!("skipping {variant:?} variant, cannot attach here: {err:#}"); diff --git a/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap new file mode 100644 index 000000000..48a216de2 --- /dev/null +++ b/crates/memtrack/tests/snapshots/c_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/c_tests.rs +expression: formatted_events +--- +[ + "Malloc { size: 1024 }", + "Malloc { size: 2048 }", + "Malloc { size: 4096 }", + "Free", + "Free", + "Free", +] diff --git a/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap new file mode 100644 index 000000000..48a216de2 --- /dev/null +++ b/crates/memtrack/tests/snapshots/c_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/c_tests.rs +expression: formatted_events +--- +[ + "Malloc { size: 1024 }", + "Malloc { size: 2048 }", + "Malloc { size: 4096 }", + "Free", + "Free", + "Free", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap new file mode 100644 index 000000000..f1d65a997 --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__nested_doubling_shared_free.snap @@ -0,0 +1,12 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 1024, has_stack: true }", + "Malloc { size: 2048, has_stack: true }", + "Malloc { size: 4096, has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap new file mode 100644 index 000000000..cf9fc935d --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_capture_disabled.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 64, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", + "Malloc { size: 192, has_stack: false }", + "Free { has_stack: false }", +] diff --git a/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap new file mode 100644 index 000000000..7cb31d31e --- /dev/null +++ b/crates/memtrack/tests/snapshots/stack_tests__stack_paths.snap @@ -0,0 +1,206 @@ +--- +source: crates/memtrack/tests/stack_tests.rs +expression: format_events(&events) +--- +[ + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 64, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", + "Malloc { size: 192, has_stack: true }", + "Free { has_stack: true }", +] diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 000000000..0a0c7b25b --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,114 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use memtrack::TrackerOptions; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::mem::discriminant; +use std::process::Command; +use tempfile::TempDir; + +fn describe_allocator_event(kind: &MemtrackEventKind) -> Option { + let description = match kind { + MemtrackEventKind::Malloc { size, stack_hash } => { + format!("Malloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::Calloc { size, stack_hash } => { + format!("Calloc {{ size: {size}, has_stack: {} }}", *stack_hash != 0) + } + MemtrackEventKind::AlignedAlloc { size, stack_hash } => format!( + "AlignedAlloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ), + MemtrackEventKind::Realloc { + size, stack_hash, .. + } => { + format!( + "Realloc {{ size: {size}, has_stack: {} }}", + *stack_hash != 0 + ) + } + MemtrackEventKind::Free { stack_hash } => { + format!("Free {{ has_stack: {} }}", *stack_hash != 0) + } + _ => return None, + }; + + Some(description) +} + +fn format_events(events: &[MemtrackEvent]) -> Vec { + const MARKER: u64 = 0xC0D5_9EED; + let has_markers = events.iter().any(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { size, .. } if size == MARKER + ) + }); + + let filtered_events = if has_markers { + shared::between_markers(events) + } else { + events + .iter() + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Free { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + ) + }) + .sorted_by_key(|e| e.timestamp) + .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) + .cloned() + .collect() + }; + + filtered_events + .iter() + .filter_map(|e| describe_allocator_event(&e.kind)) + .collect() +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::stack_paths(include_str!("../testdata/stack_paths.c"), "stack_paths", true)] +#[case::nested_doubling( + include_str!("../testdata/nested_doubling.c"), + "nested_doubling", + true +)] +#[case::nested_doubling_shared_free( + include_str!("../testdata/nested_doubling_shared_free.c"), + "nested_doubling_shared_free", + true +)] +#[case::stack_capture_disabled( + include_str!("../testdata/stack_paths.c"), + "stack_capture_disabled", + false +)] +#[test_log::test] +fn test_stack_capture( + #[case] source: &str, + #[case] name: &str, + #[case] stack_capture: bool, +) -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let options = TrackerOptions::builder() + .stack_capture(stack_capture) + .build(); + let (events, thread_handle) = shared::track_command(Command::new(binary), options)?; + + insta::assert_debug_snapshot!(name, format_events(&events)); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab97..9c3c9f188 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8e..3a432866f 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -14,14 +14,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -90,12 +100,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +121,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -150,7 +167,7 @@ fn encode_events_realistic(bencher: Bencher, n_workers: usize) { bencher.bench_local(|| { let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); output }); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c67..fed115f29 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +102,41 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + /// One executable file mapping from a native PERF_RECORD_MMAP2 record. + /// The common event header carries its address, process, and timestamp. + Mapping { + path: String, + dev: u64, + ino: u64, + file_offset: u64, + len: u64, + }, + + Stack { + // Box keeps the MemtrackEventKind enum small across millions of events. + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +166,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -139,6 +188,19 @@ mod tests { size: 40960, }, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 400, + addr: 0x400000, + kind: MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + }, ]; let artifact = MemtrackArtifact { @@ -167,21 +229,54 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Mapping { + path: "/usr/lib/libexample.so".into(), + dev: 0x0801, + ino: 0x1234, + file_offset: 0x1000, + len: 0x2000, + }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +285,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +310,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +363,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed9..8cac46f05 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 61e804de7..2cdc7d5a6 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,5 +4,6 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; +pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89b..2e48a2fd8 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -11,35 +11,43 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. #[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, - +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +57,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +109,152 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub const CURRENT_VERSION: u64 = 1; + + pub fn new(integration: (String, String), artifacts: ModuleArtifacts) -> Self { + Self { + version: Self::CURRENT_VERSION, + integration, + artifacts, + } + } + + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: MemtrackMetadata::CURRENT_VERSION, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs new file mode 100644 index 000000000..fa6fbb9dc --- /dev/null +++ b/crates/runner-shared/src/serde_pid_map.rs @@ -0,0 +1,36 @@ +//! `#[serde(with = ...)]` support for pid-keyed maps. +//! +//! JSON object keys are always strings. serde_json's direct deserializer +//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` +//! field is buffered into serde's internal `Content` first, and that path has no +//! such special case — an `i32` key then fails with `invalid type: string`. So +//! the keys are read as strings and parsed here, which works on both paths. + +use libc::pid_t; +use serde::de::{Deserializer, Error}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::HashMap; + +pub fn serialize(map: &HashMap, serializer: S) -> Result +where + V: Serialize, + S: Serializer, +{ + map.serialize(serializer) +} + +pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..b8a294d8a 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -163,7 +165,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +175,18 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); + marker_result.save_to(&results_folder).unwrap(); - Ok(exit_status) + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +196,19 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +246,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +322,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d1..9f48a81ab 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 000000000..e888668bb --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,671 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use libc::pid_t; +use runner_shared::artifacts::{ArtifactExt, MemtrackArtifact, MemtrackEventKind}; +use runner_shared::metadata::MemtrackMetadata; +use runner_shared::unwind_data::ProcessUnwindData; +use std::collections::HashMap; +use std::ops::Range; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ProcessMapping { + pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + dev: u64, + ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + file_offset: u64, + avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + timestamp: u64, +} + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata::new(integration, saved.artifacts).save_to(profile_folder) +} + +/// Read every mapping artifact in the folder. One is written per tracked root +/// process, so a run with several of them contributes several files. +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackArtifact::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + mappings.extend( + read_mappings_from_artifact(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?, + ); + } + + mappings.sort_unstable_by_key(|mapping| (mapping.pid, mapping.timestamp)); + Ok(mappings) +} + +/// Reconstruct mappings across forks because inherited perf events do not +/// synthesize mappings that already existed when a child was forked. +fn read_mappings_from_artifact(reader: R) -> Result> { + let mut timeline = MemtrackArtifact::decode_streamed(reader)? + .filter(|event| { + matches!( + &event.kind, + MemtrackEventKind::Exec + | MemtrackEventKind::Mapping { .. } + | MemtrackEventKind::Fork { .. } + ) + }) + .collect::>(); + + // Ties break so exec purges before mapping, while fork inherits that mapping. + timeline.sort_by_key(|event| { + let rank = match &event.kind { + MemtrackEventKind::Exec => 0, + MemtrackEventKind::Mapping { .. } => 1, + MemtrackEventKind::Fork { .. } => 2, + _ => unreachable!(), + }; + (event.timestamp, rank) + }); + + let mut live_mappings: HashMap> = HashMap::new(); + let mut mappings = Vec::new(); + + for event in timeline { + match event.kind { + MemtrackEventKind::Mapping { + path, + dev, + ino, + file_offset, + len, + } => { + let Some(end) = event.addr.checked_add(len) else { + debug!("Skipping mapping for {path}: address range overflows"); + continue; + }; + + let mapping = ProcessMapping { + pid: event.pid, + path, + dev, + ino, + file_offset, + avma_range: event.addr..end, + timestamp: event.timestamp, + }; + live_mappings + .entry(mapping.pid) + .or_default() + .push(mapping.clone()); + mappings.push(mapping); + } + MemtrackEventKind::Fork { parent_pid } => { + let inherited = live_mappings.get(&parent_pid).cloned().unwrap_or_default(); + let child_mappings = inherited + .into_iter() + .map(|mut mapping| { + mapping.pid = event.pid; + mapping.timestamp = event.timestamp; + mapping + }) + .collect::>(); + mappings.extend(child_mappings.iter().cloned()); + live_mappings.insert(event.pid, child_mappings); + } + MemtrackEventKind::Exec => { + live_mappings.remove(&event.pid); + } + _ => unreachable!(), + } + } + + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + let process_unwind_data = if let Some(unwind_data) = &loaded_module.unwind_data { + Some(ProcessUnwindData { + timestamp: Some(mapping.timestamp), + avma_range: mapping.avma_range.clone(), + base_avma: unwind_data.base_svma.wrapping_add(load_bias), + }) + } else { + match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + loaded_module.unwind_data = Some(unwind_data); + process_unwind_data.timestamp = Some(mapping.timestamp); + Some(process_unwind_data) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some(process_unwind_data) = process_unwind_data { + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use runner_shared::artifacts::MemtrackEvent; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + #[test] + fn sorts_interleaved_mapping_artifacts_by_pid_and_timestamp() { + let results = tempfile::tempdir().unwrap(); + + // Separate files model records drained from different per-CPU rings. + MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 20, + addr: 0x2000, + kind: MemtrackEventKind::Mapping { + path: "second-module.so".to_string(), + dev: 2, + ino: 2, + file_offset: 0x2000, + len: 0x1000, + }, + }, + MemtrackEvent { + pid: 7, + tid: 7, + timestamp: 30, + addr: 0x7000, + kind: MemtrackEventKind::Mapping { + path: "child-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0x3000, + len: 0x1000, + }, + }, + ], + } + .save_file_to(results.path(), "cpu1.MemtrackArtifact.msgpack") + .unwrap(); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 10, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: "first-module.so".to_string(), + dev: 1, + ino: 1, + file_offset: 0x1000, + len: 0x1000, + }, + }], + } + .save_file_to(results.path(), "cpu0.MemtrackArtifact.msgpack") + .unwrap(); + + let mappings = read_mappings(results.path()).unwrap(); + assert_eq!( + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.timestamp)) + .collect::>(), + vec![(7, 30), (42, 10), (42, 20)] + ); + assert_eq!(mappings[0].path, "child-module.so"); + assert_eq!(mappings[1].path, "first-module.so"); + assert_eq!(mappings[2].path, "second-module.so"); + } + #[test] + fn extracts_all_mapping_events_from_a_streamed_artifact() { + const FIRST_MODULE: &str = "first-module.so"; + const SECOND_MODULE: &str = "second-module.so"; + let artifact = MemtrackArtifact { + events: vec![ + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 10, + addr: 0, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, + }, + MemtrackEvent { + pid: 7, + tid: 8, + timestamp: 11, + addr: 0x1000, + kind: MemtrackEventKind::Mapping { + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + len: 0x2000, + }, + }, + MemtrackEvent { + pid: 9, + tid: 10, + timestamp: 12, + addr: 0x4000, + kind: MemtrackEventKind::Mapping { + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + len: 0x3000, + }, + }, + MemtrackEvent { + pid: 99, + tid: 99, + timestamp: 99, + addr: u64::MAX, + kind: MemtrackEventKind::Mapping { + path: "overflow-module.so".to_string(), + dev: 3, + ino: 3, + file_offset: 0, + len: 1, + }, + }, + MemtrackEvent { + pid: 1, + tid: 1, + timestamp: 13, + addr: 0x2000, + kind: MemtrackEventKind::Free { stack_hash: 0 }, + }, + ], + }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + + assert_eq!( + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap(), + vec![ + ProcessMapping { + pid: 7, + path: FIRST_MODULE.to_string(), + dev: 0x12_3456, + ino: 0x789, + file_offset: 0x5_2000, + avma_range: 0x1000..0x3000, + timestamp: 11, + }, + ProcessMapping { + pid: 9, + path: SECOND_MODULE.to_string(), + dev: 0x65_4321, + ino: 0xabc, + file_offset: 0x7_000, + avma_range: 0x4000..0x7000, + timestamp: 12, + }, + ] + ); + } + + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_streamed_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackArtifact { + events: vec![MemtrackEvent { + pid: 1234, + tid: 1234, + timestamp: 999, + addr: 0x5555_555a_7000, + kind: MemtrackEventKind::Mapping { + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + len: 0x109_000, + }, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MemtrackMetadata::CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } + + fn mapping_event(pid: pid_t, timestamp: u64, addr: u64, path: &str) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr, + kind: MemtrackEventKind::Mapping { + path: path.to_string(), + dev: 1, + ino: 1, + file_offset: 0, + len: 0x1000, + }, + } + } + + fn fork_event(child_pid: pid_t, parent_pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid: child_pid, + tid: child_pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Fork { parent_pid }, + } + } + + fn exec_event(pid: pid_t, timestamp: u64) -> MemtrackEvent { + MemtrackEvent { + pid, + tid: pid, + timestamp, + addr: 0, + kind: MemtrackEventKind::Exec, + } + } + + fn decode_lifecycle(events: Vec) -> Vec { + let artifact = MemtrackArtifact { events }; + let mut encoded = Vec::new(); + artifact.encode_to_writer(&mut encoded).unwrap(); + read_mappings_from_artifact(std::io::Cursor::new(encoded)).unwrap() + } + + fn mapping_summary(mappings: &[ProcessMapping]) -> Vec<(pid_t, &str, u64)> { + mappings + .iter() + .map(|mapping| (mapping.pid, mapping.path.as_str(), mapping.timestamp)) + .collect() + } + + #[test] + fn fork_without_exec_inherits_only_mappings_before_fork() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + mapping_event(100, 30, 0x2000, "after-fork.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 30), + ] + ); + } + + #[test] + fn exec_stops_inheriting_parent_mappings() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-fork.so"), + fork_event(200, 100, 20), + exec_event(200, 30), + mapping_event(100, 40, 0x2000, "after-fork.so"), + mapping_event(200, 50, 0x3000, "after-exec.so"), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-fork.so", 10), + (200, "before-fork.so", 20), + (100, "after-fork.so", 40), + (200, "after-exec.so", 50), + ] + ); + } + + #[test] + fn grandchild_inherits_transitively_from_forked_child() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "root.so"), + fork_event(200, 100, 20), + mapping_event(200, 25, 0x2000, "child.so"), + fork_event(300, 200, 30), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "root.so", 10), + (200, "root.so", 20), + (200, "child.so", 25), + (300, "root.so", 30), + (300, "child.so", 30), + ] + ); + } + + #[test] + fn equal_timestamp_events_follow_exec_mapping_fork_rank() { + let mappings = decode_lifecycle(vec![ + mapping_event(100, 10, 0x1000, "before-exec.so"), + fork_event(200, 100, 20), + mapping_event(100, 20, 0x2000, "after-exec.so"), + exec_event(100, 20), + ]); + + assert_eq!( + mapping_summary(&mappings), + vec![ + (100, "before-exec.so", 10), + (100, "after-exec.so", 20), + (200, "after-exec.so", 20), + ] + ); + } +} diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf4064..f278f07cd 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 000000000..72c2bf0ea --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,15 @@ +//! Extraction of symbols, unwind data and debug info from the ELF modules a +//! profiled process mapped, and their deduplicated on-disk layout. +//! +//! The input is a set of [`loaded_module::LoadedModule`]s, however the mappings +//! were discovered; the output is the keyed `unwind_data`/`symbols.map` files +//! plus the per-pid references that the metadata points at. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 93% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12a..3e8903ed9 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,23 +1,22 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d654070..9b917e545 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa8..5b6a04ae2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a4494..97dc7b43a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a1..fd5e1aa0c 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2e..a238cbb28 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05b..34c79e04e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7bb..990e660d9 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f90..fe5907dd0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f35..10a77b716 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e2802..724f3002e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e148..6c554024a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b031..807e10611 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f98..3956fd7d1 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2dee..edfd8e558 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca2..066c1ad0e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad056..344f4080e 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb0..7f31921e2 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; @@ -306,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b54945..1d1033b38 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ade..5f04ef8c8 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(),