From d18efd98fc96e2f2626cb5c259e4f334f3d50e11 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sat, 1 Aug 2026 23:12:29 +0800 Subject: [PATCH 01/14] Add Linux-compatible proc smaps emulation Generate Linux-shaped /proc/self/smaps and /proc//smaps snapshots from tracked guest VMAs so Redis can complete its ARM64 COW safety check under elfuse. Add growable VMA/string infrastructure and fork-aware accounting with parser regression coverage. --- Makefile | 19 + docs/internals.md | 70 ++++ mk/config.mk | 4 +- mk/tests.mk | 28 +- src/core/guest.c | 4 +- src/dynamic-array.c | 317 ++++++++++++++++ src/dynamic-array.h | 238 ++++++++++++ src/runtime/procemu.c | 459 ++++++++++++++++------- src/runtime/procemu.h | 2 +- src/string-builder.c | 283 +++++++++++++++ src/string-builder.h | 62 ++++ tests/test-dynamic-array-host.c | 109 ++++++ tests/test-matrix.sh | 5 +- tests/test-proc-smap.c | 599 +++++++++++++++++++++++++++++++ tests/test-string-builder-host.c | 182 ++++++++++ tests/test-util.h | 64 ++++ 16 files changed, 2306 insertions(+), 139 deletions(-) create mode 100644 src/dynamic-array.c create mode 100644 src/dynamic-array.h create mode 100644 src/string-builder.c create mode 100644 src/string-builder.h create mode 100644 tests/test-dynamic-array-host.c create mode 100644 tests/test-proc-smap.c create mode 100644 tests/test-string-builder-host.c diff --git a/Makefile b/Makefile index 02549c93..5bea687c 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ include mk/config.mk # Source files. SRCS := \ main.c \ + dynamic-array.c \ + string-builder.c \ core/guest.c \ core/elf.c \ core/stack.c \ @@ -192,6 +194,23 @@ $(BUILD_DIR)/test-teardown-live-vcpu-host: \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ $(HVF_LDFLAGS) +## Build the string builder host unit test (native macOS binary) +# This is a pure-C unit test; link the string builder and generic container +# implementations directly and skip the Hypervisor framework and codesign. +$(BUILD_DIR)/test-string-builder-host: \ + $(BUILD_DIR)/test-string-builder-host.o \ + $(BUILD_DIR)/string-builder.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the generic dynamic-array host unit test (native macOS binary) +$(BUILD_DIR)/test-dynamic-array-host: \ + $(BUILD_DIR)/test-dynamic-array-host.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + # Guest test binaries (cross-compiled, aarch64-linux) # Only used when GUEST_TEST_BINARIES is not set. diff --git a/docs/internals.md b/docs/internals.md index e6732d48..1aa36950 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -114,6 +114,32 @@ Key files: | `src/runtime/proctitle.c` | argv / comm rewriting for `prctl PR_SET_NAME` | | `src/debug/gdbstub.c`, `gdbstub-rsp.c`, `gdbstub-reg.c` | GDB RSP stub | +## Generic Dynamic Containers + +`src/dynamic-array.h` and `src/dynamic-array.c` provide the raw +`dynamic_array_t` used by the procfs VMA snapshot and the string builder. Capacity +is measured in element slots, while `count` is the number of logical elements. +The allocation is one contiguous block of `capacity * element_size` bytes; +both the count addition and the multiplication are checked before a growth. +Arithmetic overflow reports `EOVERFLOW`, invalid arguments report `EINVAL`, +and an allocation failure reports `ENOMEM`. Growth is transactional: on any +failure the old pointer, count, and capacity remain valid. + +The generated typed facades own only their array storage. Elements are copied +as trivially-copyable bytes, so the container does not call destructors and +does not manage pointers or other resources held by an element. `destroy` +frees the contiguous block and restores the zero state. A facade can therefore +be declared as `{0}` and initialized lazily on its first operation. + +`string_builder_t` is a thin facade over a generated `char` container. The +container count is the C-string length and excludes the terminator; its +capacity accessor reports bytes including the trailing NUL slot. Reserve and +append account for that slot, and every successful mutation restores +`data[count] == '\0'`. `string_builder_append` accepts a C string and uses its +first NUL as the end of the input. Formatted appends retain the existing +two-pass `vsnprintf` behavior and commit only the prefix through the first NUL, +matching standard C string semantics. + ## Hypervisor.framework Constraints Apple HVF imposes a handful of constraints that shape the rest of the design: @@ -768,6 +794,50 @@ under `/proc`, `/dev`, and a few Linux-expected compatibility files: - Guest cwd handling preserves a virtual `/proc` working directory even though the host operates on synthetic backing directories. +`/proc/self/smaps` and `/proc//smaps` are generated from the same tracked +VMA list as `/proc/self/maps`. The complete field set currently emitted for +each VMA is the maps header followed by these 25 fields, in this order: + +```text +Size, KernelPageSize, MMUPageSize, Rss, Pss, Pss_Dirty, +Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty, Referenced, +Anonymous, KSM, LazyFree, AnonHugePages, ShmemPmdMapped, FilePmdMapped, +Shared_Hugetlb, Private_Hugetlb, Swap, SwapPss, Locked, THPeligible, +ProtectionKey, VmFlags +``` + +`Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are +reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a +non-zero value: in a fork child, writable private anonymous VMAs report their +full VMA size because they are logically shared with the parent's CoW +snapshot. Every other numeric counter is emitted as zero, including +`THPeligible` and `ProtectionKey`. `VmFlags` is evidence-based and contains +only the permission/sharing flags represented by the tracked VMA (`rd`, `wr`, +`ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. +elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing +against a real Linux kernel should tolerate either conditional field being +absent. +These are coarse VMA-level values, not host page-residency, dirty-bit, or +proportional-sharing accounting, so they are suitable for fork-safety checks +but not precise memory profiling. `smaps_rollup` is not implemented. + +Synthetic proc directories have explicit snapshot boundaries. The backing +trees reached by opening `/proc` or `/proc/self` are materialized once, on the +first access, and their initial `stat`, `status`, `cmdline`, `maps`, and `smaps` +files remain fixed for the process lifetime. An absolute open of one of those +proc paths is intercepted directly and generates fresh content; opening the +same name through an already-open synthetic directory reads that directory's +snapshot. `/proc/self/fd` and `/proc/self/fdinfo` are rebuilt into an +independent scratch directory on every open, so each directory fd sees the +guest-fd table as it existed at that open and concurrent enumerations cannot +mutate one another. `/proc/self/task` is repopulated from the current thread +set whenever the directory is opened. These boundaries are intentional: a +directory stream is stable while it is being read, while dynamic task and fd +listings refresh only when a new directory is opened. The synthetic +`/sys/devices/system/cpu` tree follows the one-shot rule: its CPU count, +cpumask files, and `cpuN` directories are captured on first access and then +remain fixed. + Related implementation: `src/runtime/procemu.c`, `src/syscall/path.c`, `src/syscall/fs.c`, `src/syscall/proc-state.c`. diff --git a/mk/config.mk b/mk/config.mk index 969a39fc..c53ac42b 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -21,7 +21,9 @@ endif # Exclude native macOS test files from cross-compilation NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c \ tests/test-tlbi-encoder-host.c \ - tests/test-fork-ipc-protocol-host.c + tests/test-fork-ipc-protocol-host.c \ + tests/test-dynamic-array-host.c \ + tests/test-string-builder-host.c SPECIAL_TEST_SRCS := tests/test-lowbase-mem.c SPECIAL_TEST_BINS := $(BUILD_DIR)/test-lowbase-mem-200000 $(BUILD_DIR)/test-lowbase-mem-300000 diff --git a/mk/tests.mk b/mk/tests.mk index 3cbc1c58..c20e0a08 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -14,6 +14,8 @@ test-sysroot-host-fallback test-sysroot-case-exact \ test-sysroot-create-paths test-fork-ipc-protocol-host \ test-vcpu-run-hooks-host test-identity-override-host \ + test-dynamic-array-host \ + test-string-builder-host \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-fuse-alpine \ test-sysroot-nofollow test-sysroot-chdir test-sysroot-symlink-escape \ @@ -80,7 +82,9 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ $(BUILD_DIR)/test-fork-ipc-protocol-host \ $(BUILD_DIR)/test-vcpu-run-hooks-host \ $(BUILD_DIR)/test-identity-override-host \ - $(BUILD_DIR)/test-teardown-live-vcpu-host + $(BUILD_DIR)/test-teardown-live-vcpu-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v -s '$(SANITIZER_SECTIONS)' @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -92,6 +96,10 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ @$(BUILD_DIR)/test-identity-override-host @printf "\n$(BLUE)━━━ teardown live-worker accounting unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-teardown-live-vcpu-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host ## Run the unit test suite plus busybox applet validation check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @@ -99,7 +107,9 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ $(BUILD_DIR)/test-fork-ipc-protocol-host \ $(BUILD_DIR)/test-vcpu-run-hooks-host \ $(BUILD_DIR)/test-identity-override-host \ - $(BUILD_DIR)/test-teardown-live-vcpu-host + $(BUILD_DIR)/test-teardown-live-vcpu-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -111,6 +121,10 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @$(BUILD_DIR)/test-identity-override-host @printf "\n$(BLUE)━━━ teardown live-worker accounting unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-teardown-live-vcpu-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host @printf "\n$(BLUE)━━━ shebang parser unit test ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-shebang-host @printf "\n$(BLUE)━━━ proctitle argv-tail regression ━━━$(RESET)\n" @@ -770,6 +784,16 @@ test-fork-ipc-protocol-host: $(BUILD_DIR)/test-fork-ipc-protocol-host test-vcpu-run-hooks-host: $(BUILD_DIR)/test-vcpu-run-hooks-host $(BUILD_DIR)/test-vcpu-run-hooks-host +# String builder unit test +## Run the growable string builder host unit test +test-string-builder-host: $(BUILD_DIR)/test-string-builder-host + $(BUILD_DIR)/test-string-builder-host + +# Generic dynamic array unit test +## Run the raw/typed dynamic array host unit test +test-dynamic-array-host: $(BUILD_DIR)/test-dynamic-array-host + $(BUILD_DIR)/test-dynamic-array-host + # Proctitle argv-tail regression ## Run the deterministic argv-tail overshoot guard test test-proctitle-host: $(BUILD_DIR)/test-proctitle-host diff --git a/src/core/guest.c b/src/core/guest.c index 2bf395e0..addc79f4 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -2052,8 +2052,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->flags = flags; r->offset = offset; r->backing_fd = owned_backing_fd; - r->shared = (flags & 0x01) != 0; /* LINUX_MAP_SHARED = 0x01 */ - r->noreserve = (flags & 0x4000) != 0; /* LINUX_MAP_NORESERVE = 0x4000 */ + r->shared = (flags & LINUX_MAP_SHARED) != 0; + r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; guest_region_clear_overlay(r); if (name) { diff --git a/src/dynamic-array.c b/src/dynamic-array.c new file mode 100644 index 00000000..a4dd63c8 --- /dev/null +++ b/src/dynamic-array.c @@ -0,0 +1,317 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "dynamic-array.h" + +#include +#include +#include +#include + +#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t)8) + +/* Set errno for a bad argument and return a standard failure code. */ +static int dynamic_array_invalid(void) +{ + errno = EINVAL; + return -1; +} + +/* Check that the array handle is non-null and has a non-zero element size. */ +static int dynamic_array_validate(const dynamic_array_t *array) +{ + if (array == NULL || array->element_size == 0) + return dynamic_array_invalid(); + return 0; +} + +/* Return count + extra after guarding against size_t overflow. */ +static int dynamic_array_count_plus(const dynamic_array_t *array, + size_t extra, size_t *total) +{ + if (extra > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + *total = array->count + extra; + return 0; +} + +/* Compute the byte size for a number of elements with overflow protection. */ +static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, + size_t *bytes) +{ + if (count != 0 && array->element_size > SIZE_MAX / count) { + errno = EOVERFLOW; + return -1; + } + *bytes = count * array->element_size; + return 0; +} + +/* Return the offset of an in-array source span, including capacity bytes. */ +static int dynamic_array_source_offset(const dynamic_array_t *array, + const void *source, size_t bytes, + size_t *offset) +{ + if (array->data == NULL || source == NULL) + return 0; + + uintptr_t base = (uintptr_t) array->data; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t)SIZE_MAX) + return 0; + size_t start = (size_t) delta; + size_t allocation_bytes; + if (dynamic_array_bytes(array, array->capacity, &allocation_bytes) < 0) + return 0; + if (start > allocation_bytes || bytes > allocation_bytes - start) + return 0; + *offset = start; + return 1; +} + +/* Initialize only metadata; allocate storage separately when requested. */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + /* There is no allocation to fail here. Preserve storage only when the + * caller is reinitializing an array with the same element type. */ + if (array->data != NULL && array->element_size != element_size) { + errno = EINVAL; + return -1; + } + if (array->data == NULL) { + array->count = 0; + array->capacity = 0; + } + array->element_size = element_size; + return 0; +} + +/* Initialize and reserve storage for an initial number of elements. */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + size_t bytes; + if (initial_capacity != 0 && element_size > SIZE_MAX / initial_capacity) { + errno = EOVERFLOW; + return -1; + } + bytes = initial_capacity * element_size; + void *storage = NULL; + if (bytes != 0) { + storage = malloc(bytes); + if (storage == NULL) { + errno = ENOMEM; + return -1; + } + } + + free(array->data); + array->data = storage; + array->count = 0; + array->capacity = initial_capacity; + array->element_size = element_size; + return 0; +} + +/* Release backing storage and reset the array object to zero state. */ +void dynamic_array_destroy(dynamic_array_t *array) +{ + if (array == NULL) + return; + free(array->data); + *array = (dynamic_array_t){0}; +} + +/* Ensure the array has capacity for at least extra additional elements. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra) +{ + if (dynamic_array_validate(array) < 0) + return -1; + + size_t needed; + if (dynamic_array_count_plus(array, extra, &needed) < 0) + return -1; + if (needed <= array->capacity) + return 0; + + size_t new_capacity = array->capacity; + if (new_capacity == 0) + new_capacity = DYNAMIC_ARRAY_INITIAL_CAPACITY; + while (new_capacity < needed) { + if (new_capacity > SIZE_MAX / 2) { + new_capacity = needed; + break; + } + new_capacity *= 2; + } + + size_t bytes; + if (dynamic_array_bytes(array, new_capacity, &bytes) < 0) + return -1; + void *grown = realloc(array->data, bytes); + if (grown == NULL && bytes != 0) { + errno = ENOMEM; + return -1; + } + array->data = grown; + array->capacity = new_capacity; + return 0; +} + +/* Resize logical length; zero-initialize any newly visible elements. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count > array->capacity) { + size_t extra = count - array->count; + if (dynamic_array_reserve(array, extra) < 0) + return -1; + } + if (count > array->count) { + size_t old_bytes, new_bytes; + (void) dynamic_array_bytes(array, array->count, &old_bytes); + (void) dynamic_array_bytes(array, count, &new_bytes); + memset((unsigned char *) array->data + old_bytes, 0, + new_bytes - old_bytes); + } + array->count = count; + return 0; +} + +/* Append multiple elements from source memory to the end of the array. */ +int dynamic_array_append_n(dynamic_array_t *array, const void *data, + size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (dynamic_array_count_plus(array, count, &total) < 0) + return -1; + size_t bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0) + return -1; + size_t offset = 0; + int aliases = dynamic_array_source_offset(array, data, bytes, &offset); + + if (dynamic_array_reserve(array, count) < 0) + return -1; + if (aliases) + data = (const unsigned char *) array->data + offset; + size_t old_bytes; + (void) dynamic_array_bytes(array, array->count, &old_bytes); + memmove((unsigned char *) array->data + old_bytes, data, bytes); + array->count = total; + return 0; +} + +/* Insert multiple elements at index while preserving existing elements. */ +int dynamic_array_insert_n(dynamic_array_t *array, size_t index, + const void *data, size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (index > array->count) + return dynamic_array_invalid(); + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (count > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + total = array->count + count; + size_t bytes, index_bytes, tail_bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0 || + dynamic_array_bytes(array, index, &index_bytes) < 0 || + dynamic_array_bytes(array, array->count - index, &tail_bytes) < 0) + return -1; + + size_t source_offset = 0; + int aliases = dynamic_array_source_offset(array, data, bytes, + &source_offset); + void *temporary = NULL; + if (aliases) { + temporary = malloc(bytes); + if (temporary == NULL) { + errno = ENOMEM; + return -1; + } + memcpy(temporary, (const unsigned char *) array->data + source_offset, + bytes); + data = temporary; + } + + if (dynamic_array_reserve(array, count) < 0) { + free(temporary); + return -1; + } + unsigned char *base = array->data; + memmove(base + index_bytes + bytes, base + index_bytes, tail_bytes); + memcpy(base + index_bytes, data, bytes); + array->count = total; + free(temporary); + return 0; +} + +/* Append a single element by forwarding to append_n. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data) +{ + return dynamic_array_append_n(array, data, 1); +} + +/* Insert a single element by forwarding to insert_n. */ +int dynamic_array_insert_one(dynamic_array_t *array, size_t index, + const void *data) +{ + return dynamic_array_insert_n(array, index, data, 1); +} + +/* Return a mutable pointer to the index-th element, or NULL when invalid. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index) +{ + if (dynamic_array_validate(array) < 0 || index >= array->count) { + if (array != NULL && array->element_size != 0) + errno = EINVAL; + return NULL; + } + return (unsigned char *) array->data + index * array->element_size; +} + +/* Return a read-only pointer to the index-th element, or NULL when invalid. */ +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index) +{ + if (array == NULL || array->element_size == 0) { + errno = EINVAL; + return NULL; + } + if (index >= array->count) { + errno = EINVAL; + return NULL; + } + return (const unsigned char *) array->data + index * array->element_size; +} diff --git a/src/dynamic-array.h b/src/dynamic-array.h new file mode 100644 index 00000000..7e95c592 --- /dev/null +++ b/src/dynamic-array.h @@ -0,0 +1,238 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +typedef struct dynamic_array { + void *data; + size_t count; + size_t capacity; + size_t element_size; +} dynamic_array_t; + +#if defined(__GNUC__) || defined(__clang__) +#define DYNAMIC_ARRAY_INLINE static inline __attribute__((unused)) +#else +#define DYNAMIC_ARRAY_INLINE static inline +#endif + +/* Reset type metadata on a failed first-touch typed operation. */ +static inline int dynamic_array_typed_result(dynamic_array_t *array, + int was_uninitialized, + int result) +{ + if (result < 0 && was_uninitialized && array != NULL && + array->data == NULL && array->count == 0 && array->capacity == 0) + array->element_size = 0; + return result; +} + +/* Set the element size on the first typed operation and report that change. */ +static inline int dynamic_array_typed_prepare(dynamic_array_t *array, + size_t element_size) +{ + int was_uninitialized = array != NULL && array->element_size == 0; + if (was_uninitialized) + array->element_size = element_size; + return was_uninitialized; +} + +/* Initialize an array for elements of element_size bytes without allocation. */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size); + +/* Initialize and reserve initial_capacity element slots. */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity); + +/* Release storage and restore the all-zero state. */ +void dynamic_array_destroy(dynamic_array_t *array); + +/* Ensure room for extra elements beyond the current count. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra); + +/* Set the logical element count. Newly exposed elements are zeroed. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count); + +/* Append one element, or count elements when the three-argument form is + * used. The macro keeps both forms available to callers. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data); +int dynamic_array_append_n(dynamic_array_t *array, const void *data, + size_t count); +#define DYNAMIC_ARRAY_APPEND_PICK(_1, _2, _3, NAME, ...) NAME +#define dynamic_array_append(...) \ + DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ + dynamic_array_append_one, dynamic_array_append_dummy) \ + (__VA_ARGS__) + +/* Insert one element, or count elements in the four-argument form. */ +int dynamic_array_insert_one(dynamic_array_t *array, size_t index, + const void *data); +int dynamic_array_insert_n(dynamic_array_t *array, size_t index, + const void *data, size_t count); +#define DYNAMIC_ARRAY_INSERT_PICK(_1, _2, _3, _4, NAME, ...) NAME +#define dynamic_array_insert(...) \ + DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ + dynamic_array_insert_one, dynamic_array_insert_dummy) \ + (__VA_ARGS__) + +/* Return an element pointer, or NULL with errno=EINVAL for a bad index. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index); +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); + +/* Generate a small type-safe facade over the raw container. The facade owns + * no additional state; all growth and copying remains in dynamic-array.c. + */ +#define DYNAMIC_ARRAY_DEFINE(name, type) \ + typedef struct name { \ + dynamic_array_t raw; \ + } name##_t; \ + \ + /* Initialize the typed array with no preallocated storage. */ \ + DYNAMIC_ARRAY_INLINE int name##_init(name##_t *array) \ + { \ + return dynamic_array_init(array != NULL ? &array->raw : NULL, \ + sizeof(type)); \ + } \ + /* Initialize typed array and preallocate initial slots. */ \ + DYNAMIC_ARRAY_INLINE int name##_init_with_capacity(name##_t *array, \ + size_t initial_capacity) \ + { \ + return dynamic_array_init_with_capacity( \ + array != NULL ? &array->raw : NULL, sizeof(type), \ + initial_capacity); \ + } \ + /* Destroy the typed array and release backing storage. */ \ + DYNAMIC_ARRAY_INLINE void name##_destroy(name##_t *array) \ + { \ + if (array != NULL) \ + dynamic_array_destroy(&array->raw); \ + } \ + /* Reserve extra slots in the typed array. */ \ + DYNAMIC_ARRAY_INLINE int name##_reserve(name##_t *array, size_t extra) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_reserve(array != NULL ? &array->raw : NULL, extra)); \ + } \ + /* Resize typed array, zero-filling newly visible elements. */ \ + DYNAMIC_ARRAY_INLINE int name##_resize(name##_t *array, size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_resize(array != NULL ? &array->raw : NULL, count)); \ + } \ + /* Append one value through a typed pointer. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_ptr(name##_t *array, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, value, \ + 1)); \ + } \ + /* Append one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_value(name##_t *array, type value) \ + { \ + return name##_append_ptr(array, &value); \ + } \ + /* Accept either a value or a pointer to a value while retaining compile- \ + * time checking of the element type. */ \ + DYNAMIC_ARRAY_INLINE int name##_append(name##_t *array, const type *value) \ + { \ + return name##_append_ptr(array, value); \ + } \ + /* Append a typed range of values. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_n(name##_t *array, \ + const type *values, \ + size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, values, \ + count)); \ + } \ + /* Insert one typed value through a pointer form. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, \ + size_t index, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + value, 1)); \ + } \ + /* Insert one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_value(name##_t *array, \ + size_t index, \ + type value) \ + { \ + return name##_insert_ptr(array, index, &value); \ + } \ + /* Insert one typed value from a pointer argument. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert(name##_t *array, size_t index, \ + const type *value) \ + { \ + return name##_insert_ptr(array, index, value); \ + } \ + /* Insert a typed range of values at index. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, \ + size_t index, const type *values, \ + size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + values, count)); \ + } \ + /* Return a typed pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE type *name##_at(name##_t *array, size_t index) \ + { \ + return (type *) dynamic_array_at(array != NULL ? &array->raw : NULL, \ + index); \ + } \ + /* Return a typed const pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_at_const(const name##_t *array, \ + size_t index) \ + { \ + return (const type *) dynamic_array_at_const( \ + array != NULL ? &array->raw : NULL, index); \ + } \ + /* Access the underlying typed data pointer. */ \ + DYNAMIC_ARRAY_INLINE type *name##_data(name##_t *array) \ + { \ + return array != NULL ? (type *) array->raw.data : NULL; \ + } \ + /* Access the underlying typed const data pointer. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_data_const(const name##_t *array) \ + { \ + return array != NULL ? (const type *) array->raw.data : NULL; \ + } \ + /* Query current number of elements in the typed array. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_count(const name##_t *array) \ + { \ + return array != NULL ? array->raw.count : 0; \ + } \ + /* Query current allocated capacity. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_capacity(const name##_t *array) \ + { \ + return array != NULL ? array->raw.capacity : 0; \ + } diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index ed83fcc7..fb6f3681 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -10,10 +10,16 @@ * (caller falls through to real syscall). */ -/* Maximum /proc/self/maps entries. Array is sized to this; loop bounds use - * MAPS_ENTRY_MAX - 1 to leave room for safe increment. +/* Initial capacity for the transient /proc/self/maps and /proc/self/smaps VMA + * snapshot. The region tracker documents that coalescing leaves typical + * workloads at roughly 50 tracked regions (see core/guest.h), so 64 avoids an + * immediate growth in that case. The array grows as needed while mmap_lock is + * held; keep a hard ceiling tied to the guest's region tables so maps/smaps + * can enumerate every tracked VMA while keeping transient snapshot memory + * bounded. */ -#define MAPS_ENTRY_MAX 256 +#define MAPS_ENTRY_INITIAL_CAP 64 +#define MAPS_ENTRY_MAX (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED) /* Bound the transient host-PID snapshot used by /proc/net enumeration. This * is an output-work limit, not the dynamically growing lifecycle-table cap. @@ -51,6 +57,7 @@ #include #include +#include "string-builder.h" #include "utils.h" #include "debug/log.h" @@ -97,55 +104,70 @@ typedef struct { char name[64]; } maps_entry_t; -static void maps_entry_insert(maps_entry_t *entries, - int *nentries, - uint64_t start, - uint64_t end, - int prot, - int flags, - uint64_t offset, - const char *name) +/* A growable VMA snapshot. The generated facade keeps element-size and + * allocation bookkeeping in the generic array implementation. */ +DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) + +static int maps_entries_insert_sorted(maps_entries_t *entries, + uint64_t start, + uint64_t end, + int prot, + int flags, + uint64_t offset, + const char *name) { - if (*nentries >= MAPS_ENTRY_MAX || end <= start) - return; - - int i = *nentries; - while (i > 0 && entries[i - 1].start > start) { - entries[i] = entries[i - 1]; - i--; + if (end <= start) + return 0; + if (maps_entries_count(entries) >= MAPS_ENTRY_MAX) { + errno = ENOMEM; + return -1; } + if (maps_entries_count(entries) == 0 && + maps_entries_reserve(entries, MAPS_ENTRY_INITIAL_CAP) < 0) + return -1; - maps_entry_t *e = &entries[i]; - e->start = start; - e->end = end; - e->prot = prot; - e->flags = flags; - e->offset = offset; + maps_entry_t value = { + .start = start, + .end = end, + .prot = prot, + .flags = flags, + .offset = offset, + }; if (name && name[0]) - str_copy_trunc(e->name, name, sizeof(e->name)); + str_copy_trunc(value.name, name, sizeof(value.name)); else - e->name[0] = '\0'; - (*nentries)++; + value.name[0] = '\0'; + + size_t i = maps_entries_count(entries); + while (i > 0 && maps_entries_at(entries, i - 1)->start > start) { + i--; + } + return maps_entries_insert(entries, i, &value); } -static void maps_entries_merge_adjacent(maps_entry_t *entries, int *nentries) +static void maps_entries_merge_adjacent(maps_entries_t *entries) { - if (*nentries <= 1) + size_t count = maps_entries_count(entries); + if (count <= 1) return; - int out = 0; - for (int i = 1; i < *nentries; i++) { - if (entries[i].start == entries[out].end && - entries[i].prot == entries[out].prot && - entries[i].flags == entries[out].flags && - entries[i].offset == entries[out].offset && - strcmp(entries[i].name, entries[out].name) == 0) { - entries[out].end = entries[i].end; + size_t out = 0; + for (size_t i = 1; i < count; i++) { + maps_entry_t *current = maps_entries_at(entries, i); + maps_entry_t *previous = maps_entries_at(entries, out); + if (current->start == previous->end && + current->prot == previous->prot && + current->flags == previous->flags && + current->offset == previous->offset && + strcmp(current->name, previous->name) == 0) { + previous->end = current->end; continue; } - entries[++out] = entries[i]; + ++out; + if (out != i) + *maps_entries_at(entries, out) = *current; } - *nentries = out + 1; + (void)maps_entries_resize(entries, out + 1); } /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily @@ -445,7 +467,8 @@ static void proc_tmpdir_cleanup(void) /* Remove known files inside // and / */ char path[256]; - const char *files[] = {"stat", "status", "cmdline", "maps", "exe", NULL}; + const char *files[] = {"stat", "status", "cmdline", "maps", "smaps", + "exe", NULL}; char piddir[160]; /* Reconstruct pid subdir by scanning for the first numeric entry */ @@ -1234,6 +1257,7 @@ static const char *ensure_proc_tmpdir(const guest_t *g) populate_proc_snapshot(g, piddir, "status", "/proc/self/status"); populate_proc_snapshot(g, piddir, "cmdline", "/proc/self/cmdline"); populate_proc_snapshot(g, piddir, "maps", "/proc/self/maps"); + populate_proc_snapshot(g, piddir, "smaps", "/proc/self/smaps"); /* Create task subdirectory for /proc/self/task enumeration */ char taskdir[128]; @@ -2330,58 +2354,54 @@ static int pty_open_master(int linux_flags) return master; } -/* Emit /proc/self/maps into a synthetic fd. Merges contiguous regions[] runs - * that came from one mmap, then folds in preannounced[] shadow entries whose - * advertised interval is not yet fully covered by live regions. +/* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges + * contiguous regions[] runs that came from one mmap, then folds in + * preannounced[] shadow entries whose advertised interval is not yet fully + * covered by live regions. * - * Returns a host fd, or -1 on error. Split out of proc_intercept_open to keep - * that dispatcher readable. + * The region and preannounced arrays are mutable from guest mmap/mprotect/ + * munmap operations. Snapshot and merge while mmap_lock is held, then release + * it before formatting output so readers never observe a torn VMA or metadata + * record and output generation does not block page-table mutations. */ -static int proc_open_self_maps(const guest_t *g) +static int proc_build_maps_entries(const guest_t *g, + maps_entries_t *entries_out) { - /* Heap-allocated: guest threads run on host pthreads whose stacks do not - * comfortably hold a 16KiB frame. - */ - const size_t bufsz = 16384; - char *buf = malloc(bufsz); - if (!buf) - return -1; - int off = 0; - - /* Build a flat array of (va_start, va_end, prot, flags, offset, name) from - * regions[] plus /proc/self/maps-only preannounced[] entries. - * preannounced[] is intentionally NOT consulted by mmap conflict detection, - * so advertise-only Rosetta/JIT regions do not trip MAP_FIXED_NOREPLACE - * with -EEXIST. - * - * entries is heap-allocated: MAPS_ENTRY_MAX * sizeof(maps_entry_t) is - * ~24KiB, too large for a guest-thread pthread stack. - */ - maps_entry_t *entries = calloc(MAPS_ENTRY_MAX, sizeof(*entries)); - if (!entries) { - free(buf); + if (!g || !entries_out) { + errno = EINVAL; return -1; } - int nentries = 0; + + maps_entries_t entries = {0}; + int result = -1; + + pthread_mutex_lock(&mmap_lock); /* Convert regions[] to maps entries. regions[] is already sorted by start - * address; merge contiguous runs that came from one mmap. + * address. The MAP_SHARED/MAP_ANONYMOUS/MAP_NORESERVE bits are preserved + * in r->flags, which is the single source of truth for the proc snapshot. */ - for (int i = 0; i < g->nregions && nentries < MAPS_ENTRY_MAX; i++) { + int nregions = g->nregions; + if (nregions < 0) + nregions = 0; + if (nregions > GUEST_MAX_REGIONS) + nregions = GUEST_MAX_REGIONS; + for (int i = 0; i < nregions; i++) { const guest_region_t *r = &g->regions[i]; uint64_t start = r->start & ~0xFFFULL; - uint64_t end = (r->end + 0xFFF) & ~0xFFFULL; - - if (nentries > 0 && entries[nentries - 1].end == start && - entries[nentries - 1].prot == r->prot && - entries[nentries - 1].flags == r->flags && - entries[nentries - 1].offset == r->offset && - !strcmp(entries[nentries - 1].name, r->name)) { - entries[nentries - 1].end = end; + uint64_t end = (r->end + 0xFFFULL) & ~0xFFFULL; + size_t count = maps_entries_count(&entries); + maps_entry_t *last = + count > 0 ? maps_entries_at(&entries, count - 1) : NULL; + if (last != NULL && last->end == start && last->prot == r->prot && + last->flags == r->flags && last->offset == r->offset && + !strcmp(last->name, r->name)) { + last->end = end; continue; } - maps_entry_insert(entries, &nentries, start, end, r->prot, r->flags, - r->offset, r->name); + if (maps_entries_insert_sorted(&entries, start, end, r->prot, r->flags, + r->offset, r->name) < 0) + goto out_unlock; } /* Add preannounced entries only while they still have an uncovered tail. @@ -2390,12 +2410,17 @@ static int proc_open_self_maps(const guest_t *g) * split VMAs. A partial union must stay visible because some * reserved-but-not-realized span remains to advertise. */ - for (int i = 0; i < g->npreannounced && nentries < MAPS_ENTRY_MAX; i++) { + int npreannounced = g->npreannounced; + if (npreannounced < 0) + npreannounced = 0; + if (npreannounced > GUEST_MAX_PREANNOUNCED) + npreannounced = GUEST_MAX_PREANNOUNCED; + for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; bool shadowed = false; uint64_t covered_end = r->start; - for (int j = 0; j < g->nregions; j++) { + for (int j = 0; j < nregions; j++) { const guest_region_t *live = &g->regions[j]; if (live->end <= covered_end) @@ -2413,61 +2438,226 @@ static int proc_open_self_maps(const guest_t *g) if (shadowed) continue; - maps_entry_insert(entries, &nentries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, r->flags, - r->offset, r->name); + if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, + (r->end + 0xFFFULL) & ~0xFFFULL, + r->prot, r->flags, r->offset, + r->name) < 0) + goto out_unlock; + } + maps_entries_merge_adjacent(&entries); + result = (int)maps_entries_count(&entries); + +out_unlock: + pthread_mutex_unlock(&mmap_lock); + if (result < 0) { + maps_entries_destroy(&entries); + errno = ENOMEM; + return -1; + } + *entries_out = entries; + return (int)maps_entries_count(&entries); +} + +/* Format the common VMA header. The maps and smaps header must stay byte-for- + * byte compatible so consumers can use either file interchangeably. + */ +static int proc_format_maps_header(const maps_entry_t *e, + char *header, + size_t headersz) +{ + char perms[5]; + perms[0] = (e->prot & LINUX_PROT_READ) ? 'r' : '-'; + perms[1] = (e->prot & LINUX_PROT_WRITE) ? 'w' : '-'; + perms[2] = (e->prot & LINUX_PROT_EXEC) ? 'x' : '-'; + perms[3] = (e->flags & LINUX_MAP_SHARED) ? 's' : 'p'; + perms[4] = '\0'; + + int header_len = snprintf( + header, headersz, "%llx-%llx %s %08llx 00:00 0", + (unsigned long long) e->start, (unsigned long long) e->end, perms, + (unsigned long long) e->offset); + if (header_len < 0) + return -1; + if ((size_t) header_len >= headersz) + header_len = (int) headersz - 1; + + if (e->name[0]) { + while (header_len < MAPS_NAME_COLUMN && + (size_t) header_len < headersz - 1) + header[header_len++] = ' '; + int n = snprintf(header + header_len, headersz - (size_t) header_len, + "%s", e->name); + if (n > 0) { + if ((size_t) n >= headersz - (size_t) header_len) + n = (int) (headersz - (size_t) header_len - 1); + header_len += n; + } + } else if ((size_t) header_len < headersz - 1) { + header[header_len++] = ' '; } - maps_entries_merge_adjacent(entries, &nentries); + header[header_len] = '\0'; + return header_len; +} + +/* Release the resources shared by maps/smaps output without hiding the errno + * from the operation that produced result. + */ +static int proc_finish_maps_output(int result, + maps_entries_t *entries, + string_builder_t *builder) +{ + int saved_errno = errno; + maps_entries_destroy(entries); + string_builder_destroy(builder); + errno = saved_errno; + return result; +} + +/* Emit /proc/self/maps into a synthetic fd. Addresses are page-aligned and + * output matches the Linux maps header format. + */ +static int proc_open_self_maps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 256; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; /* Emit lines after merging so buffer accounting is centralized. */ - for (int i = 0; i < nentries && off < (int) bufsz - 256; i++) { - const maps_entry_t *e = &entries[i]; - char perms[5]; - perms[0] = (e->prot & 0x1) ? 'r' : '-'; - perms[1] = (e->prot & 0x2) ? 'w' : '-'; - perms[2] = (e->prot & 0x4) ? 'x' : '-'; - perms[3] = (e->flags & 0x01) ? 's' : 'p'; - perms[4] = '\0'; - - /* Format matches real Linux /proc//maps exactly: - * %lx-%lx %s %08lx %02x:%02x %lu %s\n - * Verified against strace in a real Lima VZ VM. - */ + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); char line[256]; - int lineoff = - snprintf(line, sizeof(line), "%llx-%llx %s %08llx 00:00 0", - (unsigned long long) e->start, (unsigned long long) e->end, - perms, (unsigned long long) e->offset); - /* Cap lineoff to buffer size (snprintf may return more than available - * on truncation) - */ - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - if (e->name[0]) { - while (lineoff < MAPS_NAME_COLUMN && - lineoff < (int) sizeof(line) - 1) - line[lineoff++] = ' '; - int n = - snprintf(line + lineoff, sizeof(line) - lineoff, "%s", e->name); - if (n > 0) - lineoff += n; - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - } else if (lineoff < (int) sizeof(line) - 1) { - line[lineoff++] = ' '; - } - int wrote = snprintf(buf + off, bufsz - off, "%.*s\n", lineoff, line); - if (wrote > 0 && off + wrote < (int) bufsz) - off += wrote; - else - break; /* Stop before truncating a maps line. */ + int line_len = proc_format_maps_header(e, line, sizeof(line)); + if (line_len < 0 || + string_builder_appendf(&builder, "%.*s\n", line_len, line) < 0) + goto out; } - log_debug("/proc/self/maps (%d bytes):\n%.*s", off, off, buf); - int fd = proc_synthetic_fd(buf, off); - free(entries); - free(buf); - return fd; + log_debug("/proc/self/maps (%zu bytes):\n%.*s", string_builder_length(&builder), + (int)string_builder_length(&builder), + string_builder_data_const(&builder) ? string_builder_data_const(&builder) + : ""); + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); + +out: + return proc_finish_maps_output(result, &entries, &builder); +} + +/* Emit a Linux-shaped /proc/self/smaps approximation. The runtime tracks + * guest VMAs and logical fork snapshots, but it cannot observe kernel page + * residency or dirty bits on the host. For a fork child, writable private + * anonymous VMAs therefore report their full VMA size as Shared_Dirty; all + * other fields that require kernel page accounting are stable zeroes. + */ +static int proc_open_self_smaps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 768; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; + + bool fork_child = proc_get_ppid() != 0; + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + char header[256]; + int header_len = proc_format_maps_header(e, header, sizeof(header)); + if (header_len < 0) + goto out; + + uint64_t size_kb = (e->end - e->start) / 1024; + bool anonymous = (e->flags & LINUX_MAP_ANONYMOUS) != 0; + bool shared = (e->flags & LINUX_MAP_SHARED) != 0; + bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; + bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && + anonymous && !shared; + bool logical_shared_dirty = fork_child && private_anon && + (e->prot & LINUX_PROT_WRITE); + uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; + + char vmflags[64]; + size_t vmflags_len = 0; +#define APPEND_VMFLAG(flag) \ + do { \ + const char *token = (flag); \ + size_t token_len = strlen(token); \ + if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ + vmflags[vmflags_len++] = ' '; \ + memcpy(vmflags + vmflags_len, token, token_len); \ + vmflags_len += token_len; \ + } \ + } while (0) + /* Only flags directly evidenced by the tracked VMA are reported. In + * particular, do not invent Linux max-permission/accounting flags + * (mr/mw/me/ac/sd/etc.) that the emulator cannot observe. + */ + if (e->prot & LINUX_PROT_READ) + APPEND_VMFLAG("rd"); + if (e->prot & LINUX_PROT_WRITE) + APPEND_VMFLAG("wr"); + if (e->prot & LINUX_PROT_EXEC) + APPEND_VMFLAG("ex"); + if (shared) + APPEND_VMFLAG("sh"); + if (noreserve) + APPEND_VMFLAG("nr"); +#undef APPEND_VMFLAG + vmflags[vmflags_len] = '\0'; + + if (string_builder_appendf( + &builder, + "%.*s\n" + "Size: %llu kB\n" + "KernelPageSize: 4 kB\n" + "MMUPageSize: 4 kB\n" + "Rss: 0 kB\n" + "Pss: 0 kB\n" + "Pss_Dirty: 0 kB\n" + "Shared_Clean: 0 kB\n" + "Shared_Dirty: %llu kB\n" + "Private_Clean: 0 kB\n" + "Private_Dirty: 0 kB\n" + "Referenced: 0 kB\n" + "Anonymous: 0 kB\n" + "KSM: 0 kB\n" + "LazyFree: 0 kB\n" + "AnonHugePages: 0 kB\n" + "ShmemPmdMapped: 0 kB\n" + "FilePmdMapped: 0 kB\n" + "Shared_Hugetlb: 0 kB\n" + "Private_Hugetlb: 0 kB\n" + "Swap: 0 kB\n" + "SwapPss: 0 kB\n" + "Locked: 0 kB\n" + "THPeligible: 0\n" + "ProtectionKey: 0\n" + "VmFlags:%s\n", + header_len, header, (unsigned long long) size_kb, + (unsigned long long) shared_dirty_kb, vmflags) < 0) + goto out; + } + + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); + +out: + return proc_finish_maps_output(result, &entries, &builder); } /* Emit /proc/meminfo from host sysctl (HW_MEMSIZE) plus mach vm_statistics64, @@ -3076,6 +3266,12 @@ int proc_intercept_open(const guest_t *g, if (!strcmp(path, "/proc/self/maps")) return proc_open_self_maps(g); + /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata + * and the coarse fork Shared_Dirty compatibility signal. + */ + if (!strcmp(path, "/proc/self/smaps")) + return proc_open_self_smaps(g); + /* /proc/uptime -> synthetic uptime in seconds. Uses sysctl(KERN_BOOTTIME), * same as sys_sysinfo() in syscall/sys.c. Idle time is 0 (no meaningful * macOS equivalent). @@ -3680,6 +3876,7 @@ int proc_intercept_stat(const char *path, struct stat *st) "/proc/self/status", "/proc/self/cmdline", "/proc/self/maps", + "/proc/self/smaps", "/proc/self/exe", "/proc/self/environ", "/proc/self/auxv", diff --git a/src/runtime/procemu.h b/src/runtime/procemu.h index 32d8767a..3e66ad0e 100644 --- a/src/runtime/procemu.h +++ b/src/runtime/procemu.h @@ -24,7 +24,7 @@ #define PROC_NOT_INTERCEPTED (-2) /* Intercept openat for /proc and /dev paths. The guest_t pointer is needed to - * generate /proc/self/maps from region data. + * generate /proc/self/maps and /proc/self/smaps from region data. * Returns a host fd on match (caller should fd_alloc it), -1 on error with * errno set, or PROC_NOT_INTERCEPTED if the path is not intercepted. */ diff --git a/src/string-builder.c b/src/string-builder.c new file mode 100644 index 00000000..5cd994d2 --- /dev/null +++ b/src/string-builder.c @@ -0,0 +1,283 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "string-builder.h" + +#include +#include +#include +#include +#include + +/* Set EILSEQ for invalid input and report failure. */ +static int string_builder_invalid(void) +{ + errno = EILSEQ; + return -1; +} + +/* Initialize storage and establish an empty, NUL-terminated builder. */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity) +{ + if (builder == NULL) + return string_builder_invalid(); + if (initial_capacity == 0) { + string_builder_destroy(builder); + return 0; + } + if (string_builder_storage_init_with_capacity(&builder->storage, + initial_capacity) < 0) + return -1; + string_builder_data(builder)[0] = '\0'; + return 0; +} + +/* Release storage and reset the logical length. */ +void string_builder_destroy(string_builder_t *builder) +{ + if (builder == NULL) + return; + string_builder_storage_destroy(&builder->storage); +} + +/* Reserve enough capacity for extra bytes after the current contents. */ +int string_builder_reserve(string_builder_t *builder, size_t extra) +{ + if (builder == NULL) + return string_builder_invalid(); + if (extra == 0 && string_builder_storage_count(&builder->storage) == 0 && + string_builder_capacity(builder) == 0) + return 0; + + /* The dynamic array counts payload elements. Reserve one additional char + * for the string builder's trailing NUL. */ + if (extra == SIZE_MAX) { + errno = EOVERFLOW; + return -1; + } + if (string_builder_storage_reserve(&builder->storage, extra + 1) < 0) + return -1; + string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = + '\0'; + return 0; +} + +/* Locate a source span that aliases the builder allocation. The offset must be + * captured before reserve because reserve may move the allocation. */ +static int string_builder_source_offset(const string_builder_t *builder, + const void *source, size_t length, + size_t *offset) +{ + const char *base_ptr = string_builder_data_const(builder); + size_t capacity = string_builder_capacity(builder); + if (base_ptr == NULL || source == NULL) + return 0; + uintptr_t base = (uintptr_t) (const void *) base_ptr; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t) SIZE_MAX) + return 0; + size_t start = (size_t) delta; + if (start > capacity || length > capacity - start) + return 0; + *offset = start; + return 1; +} + +/* Commit string bytes through the generic array and restore the terminator. */ +static int string_builder_commit_append(string_builder_t *builder, + const char *data, size_t len) +{ + if (string_builder_storage_append_n(&builder->storage, data, len) < 0) + return -1; + string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = + '\0'; + return 0; +} + +/* Append a C string and keep the builder NUL-terminated. */ +int string_builder_append(string_builder_t *builder, const char *text) +{ + if (builder == NULL || text == NULL) + return string_builder_invalid(); + + size_t len = strlen(text); + if (len == 0) + return 0; + + size_t source_offset = 0; + int aliases = string_builder_source_offset(builder, text, len, + &source_offset); + if (string_builder_reserve(builder, len) < 0) + return -1; + if (aliases) + text = string_builder_data_const(builder) + source_offset; + return string_builder_commit_append(builder, text, len); +} + +/* Convert a formatting failure into the module's documented errno values. */ +static int string_builder_format_failure(void) +{ + if (errno != EOVERFLOW && errno != EILSEQ) + errno = EILSEQ; + return -1; +} + +/* Call vsnprintf with a copied argument list for the current sizing pass. */ +static int string_builder_vsnprintf(char *destination, size_t available, + const char *format, va_list arguments) +{ + va_list pass; + va_copy(pass, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + int result = vsnprintf(destination, available, format, pass); +#pragma clang diagnostic pop + va_end(pass); + return result; +} + +typedef enum string_builder_format_status { + STRING_BUILDER_FORMAT_ERROR = -1, + STRING_BUILDER_FORMAT_FIT = 0, + STRING_BUILDER_FORMAT_TRUNCATED = 1 +} string_builder_format_status_t; + +/* Ensure the builder has a writable tail and capture its current length. */ +static int string_builder_prepare_appendf(string_builder_t *builder, + size_t *old_len) +{ + if (string_builder_capacity(builder) == 0 && + string_builder_reserve(builder, 1) < 0) + return -1; + + char *data = string_builder_data(builder); + size_t capacity = string_builder_capacity(builder); + *old_len = string_builder_storage_count(&builder->storage); + if (data == NULL || *old_len >= capacity) + return string_builder_invalid(); + return 0; +} + +/* Format into the current tail and report whether the result was truncated. */ +static string_builder_format_status_t +string_builder_try_format(string_builder_t *builder, size_t old_len, + const char *format, va_list arguments, + size_t *written) +{ + char *data = string_builder_data(builder); + size_t available = string_builder_capacity(builder) - old_len; + int formatted = string_builder_vsnprintf(data + old_len, available, format, + arguments); + if (formatted < 0) + return STRING_BUILDER_FORMAT_ERROR; + + size_t visible = strlen(data + old_len); + if ((size_t) formatted < available || visible < available - 1) { + *written = visible; + return STRING_BUILDER_FORMAT_FIT; + } + + *written = (size_t) formatted; + return STRING_BUILDER_FORMAT_TRUNCATED; +} + +/* Restore the terminator after a failed formatting attempt. */ +static void string_builder_rollback_appendf(string_builder_t *builder, + size_t old_len) +{ + char *data = string_builder_data(builder); + if (data != NULL && old_len < string_builder_capacity(builder)) + data[old_len] = '\0'; +} + +/* Append printf-formatted text, growing and retrying when the first pass is + * truncated. + */ +int string_builder_appendf(string_builder_t *builder, const char *format, ...) +{ + int result = -1; + int saved_errno; + size_t old_len = 0; + size_t written = 0; + string_builder_format_status_t status; + va_list arguments; + + if (builder == NULL || format == NULL) + return string_builder_invalid(); + + saved_errno = errno; + va_start(arguments, format); + + if (string_builder_prepare_appendf(builder, &old_len) < 0) + goto out; + + status = string_builder_try_format(builder, old_len, format, arguments, + &written); + if (status == STRING_BUILDER_FORMAT_ERROR) { + string_builder_format_failure(); + goto rollback; + } + if (status == STRING_BUILDER_FORMAT_TRUNCATED) { + if (string_builder_reserve(builder, written) < 0) + goto rollback; + + status = string_builder_try_format(builder, old_len, format, arguments, + &written); + if (status == STRING_BUILDER_FORMAT_ERROR) { + string_builder_format_failure(); + goto rollback; + } + if (status == STRING_BUILDER_FORMAT_TRUNCATED) { + errno = EOVERFLOW; + goto rollback; + } + } + + if (string_builder_commit_append( + builder, string_builder_data(builder) + old_len, written) < 0) + goto rollback; + + errno = saved_errno; + result = 0; + goto out; + +rollback: + string_builder_rollback_appendf(builder, old_len); + +out: + va_end(arguments); + return result; +} + +/* Return mutable storage for the builder, if it has been allocated. */ +char *string_builder_data(string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_data(&builder->storage) : NULL; +} + +/* Return const storage for the builder, if it has been allocated. */ +const char *string_builder_data_const(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_data_const(&builder->storage) + : NULL; +} + +/* Return the number of data bytes currently stored. */ +size_t string_builder_length(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_count(&builder->storage) : 0; +} + +/* Return allocated capacity in bytes, including the terminating NUL. */ +size_t string_builder_capacity(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_capacity(&builder->storage) : 0; +} diff --git a/src/string-builder.h b/src/string-builder.h new file mode 100644 index 00000000..8feb6ae3 --- /dev/null +++ b/src/string-builder.h @@ -0,0 +1,62 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "dynamic-array.h" + +DYNAMIC_ARRAY_DEFINE(string_builder_storage, char) + +/* The storage representation is an implementation detail; callers should use + * the accessors below. The generated count is the string length; capacity is + * measured in bytes and includes the trailing NUL slot. + */ +typedef struct string_builder { + string_builder_storage_t storage; +} string_builder_t; + +/* Initialize a builder, reserving initial_capacity bytes including the NUL. + * A zero capacity leaves storage unallocated for lazy growth. Returns 0 on + * success or -1 with errno set on invalid input or allocation failure. + */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity); + +/* Release the builder storage and reset its length. Safe to call with NULL. */ +void string_builder_destroy(string_builder_t *builder); + +/* Ensure room for extra string bytes plus the terminating NUL. */ +int string_builder_reserve(string_builder_t *builder, size_t extra); + +/* Append a C string, accepting aliases into the builder's storage. As with + * standard C string functions, the first NUL terminates the input. A NULL + * pointer is invalid. The resulting C string is always NUL-terminated. + */ +int string_builder_append(string_builder_t *builder, const char *text); + +/* Append formatted text using printf-style arguments. A formatted NUL byte + * terminates the appended C string prefix. + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((format(printf, 2, 3))) +#endif +int string_builder_appendf(string_builder_t *builder, const char *format, ...); + +/* Return mutable builder data, or NULL when builder is NULL or unallocated. + * Callers must preserve the no-embedded-NUL C-string invariant. + */ +char *string_builder_data(string_builder_t *builder); + +/* Return read-only builder data, or NULL when builder is NULL or unallocated. */ +const char *string_builder_data_const(const string_builder_t *builder); + +/* Return the number of data bytes currently stored, excluding the NUL. */ +size_t string_builder_length(const string_builder_t *builder); + +/* Return allocated capacity in bytes, including space for the NUL. */ +size_t string_builder_capacity(const string_builder_t *builder); diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c new file mode 100644 index 00000000..73938842 --- /dev/null +++ b/tests/test-dynamic-array-host.c @@ -0,0 +1,109 @@ +/* + * Native-host unit tests for the generic dynamic array. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#include "dynamic-array.h" + +typedef struct odd { + unsigned char tag; + uint32_t value; +} odd_t; + +DYNAMIC_ARRAY_DEFINE(int_array, int) +DYNAMIC_ARRAY_DEFINE(odd_array, odd_t) + +static void test_zero_and_init(void) +{ + int_array_t values = {0}; + int value = 7; + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + assert(int_array_append(&values, &value) == 0); + assert(int_array_count(&values) == 1); + assert(*int_array_at(&values, 0) == 7); + int_array_destroy(&values); + assert(int_array_data(&values) == NULL); + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_capacity(&values) >= 4); + int_array_destroy(&values); +} + +static void test_growth_insert_and_stride(void) +{ + odd_array_t values = {0}; + odd_t first = {1, 11}; + odd_t second = {2, 22}; + odd_t middle = {3, 33}; + assert(odd_array_append(&values, &first) == 0); + assert(odd_array_append(&values, &second) == 0); + assert(odd_array_insert(&values, 1, &middle) == 0); + assert(odd_array_count(&values) == 3); + assert(odd_array_at(&values, 1)->tag == 3); + assert(odd_array_at(&values, 2)->value == 22); + assert(odd_array_capacity(&values) >= 3); + odd_array_destroy(&values); +} + +static void test_alias_and_resize(void) +{ + int_array_t values = {0}; + int initial[] = {1, 2, 3, 4}; + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_append_n(&values, initial, 4) == 0); + int *old_data = int_array_data(&values); + int *alias = int_array_at(&values, 1); + assert(int_array_append(&values, alias) == 0); + assert(int_array_data(&values) != old_data || int_array_capacity(&values) > 4); + assert(int_array_count(&values) == 5); + assert(*int_array_at(&values, 4) == 2); + assert(int_array_resize(&values, 8) == 0); + for (size_t i = 5; i < 8; i++) + assert(*int_array_at(&values, i) == 0); + assert(int_array_resize(&values, 2) == 0); + int_array_destroy(&values); +} + +static void test_invalid_and_overflow(void) +{ + int_array_t values = {0}; + int value = 9; + errno = 0; + assert(int_array_insert(&values, 1, &value) == -1); + assert(errno == EINVAL); + errno = 0; + assert(int_array_append_n(&values, &value, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + + dynamic_array_t raw = {0}; + errno = 0; + assert(dynamic_array_init(&raw, 0) == -1); + assert(errno == EINVAL); + assert(dynamic_array_init(&raw, sizeof(uint64_t)) == 0); + errno = 0; + assert(dynamic_array_reserve(&raw, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(raw.data == NULL && raw.count == 0 && raw.capacity == 0); + dynamic_array_destroy(&raw); +} + +int main(void) +{ + test_zero_and_init(); + test_growth_insert_and_stride(); + test_alias_and_resize(); + test_invalid_and_overflow(); + puts("test-dynamic-array-host: PASS"); + return 0; +} diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 8bde500a..a5cbdb11 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -667,6 +667,7 @@ run_unit_tests() test_rc "$runner" "test-procfs-exec" 0 "$bindir/test-procfs-exec" test_rc "$runner" "test-proc-limits" 0 "$bindir/test-proc-limits" test_rc "$runner" "test-proc-fidelity" 0 "$bindir/test-proc-fidelity" + test_rc "$runner" "test-proc-smap" 0 "$bindir/test-proc-smap" printf "\nNetwork\n" test_check "$runner" "test-net" "0 failed" "$bindir/test-net" @@ -1231,8 +1232,8 @@ run_suite() # observed counts diverge. apple-unknown is the fallback row for SoC strings the # detector does not recognize yet. EXPECTED_BASELINES=( - "elfuse-aarch64|238|0" - "qemu-aarch64|218|0" + "elfuse-aarch64|239|0" + "qemu-aarch64|219|0" "elfuse-x86_64:apple-m1-m2|71|0" "elfuse-x86_64:apple-m3-plus|71|0" "elfuse-x86_64:apple-unknown|71|0" diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c new file mode 100644 index 00000000..99786148 --- /dev/null +++ b/tests/test-proc-smap.c @@ -0,0 +1,599 @@ +/* + * Generic /proc//smaps parser and accounting regression test. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" +#include "test-util.h" + +int passes = 0, fails = 0; + +typedef struct { + uintptr_t start; + uintptr_t end; + unsigned long long offset; + char perms[5]; + unsigned long long size_kb; + unsigned long long kernel_page_kb; + unsigned long long mmu_page_kb; + unsigned long long shared_dirty_kb; + bool have_size; + bool have_shared_dirty; + bool have_vmflags; + bool vmflags_wr; +} smaps_vma_t; + +static const char *const smaps_fields[] = { + "Size:", "KernelPageSize:", "MMUPageSize:", + "Rss:", "Pss:", "Pss_Dirty:", + "Shared_Clean:", "Shared_Dirty:", "Private_Clean:", + "Private_Dirty:", "Referenced:", "Anonymous:", + "KSM:", "LazyFree:", "AnonHugePages:", + "ShmemPmdMapped:", + "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", + "Swap:", "SwapPss:", "Locked:", + "THPeligible:", "ProtectionKey:", "VmFlags:", +}; + +#define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) +#define SMAPS_KB_FIELD_COUNT 22 + +typedef struct { + smaps_vma_t *vmas; + size_t count; +} smaps_info_t; + +static const char *skip_space(const char *p) +{ + while (*p && isspace((unsigned char) *p)) + p++; + return p; +} + +static bool parse_token(const char **cursor, char *token, size_t token_size) +{ + const char *p = skip_space(*cursor); + const char *start = p; + while (*p && !isspace((unsigned char) *p)) + p++; + size_t len = (size_t) (p - start); + if (len == 0 || len >= token_size) + return false; + memcpy(token, start, len); + token[len] = '\0'; + *cursor = p; + return true; +} + +static bool parse_unsigned_token(const char *token, int base, + unsigned long long *value) +{ + if (!token[0]) + return false; + for (const unsigned char *p = (const unsigned char *) token; *p; p++) { + if (base == 16 ? !isxdigit(*p) : !isdigit(*p)) + return false; + } + + errno = 0; + char *end = NULL; + unsigned long long n = strtoull(token, &end, base); + if (errno == ERANGE || end == token || *end != '\0') + return false; + *value = n; + return true; +} + +static bool parse_device_token(const char *token) +{ + const char *colon = strchr(token, ':'); + if (!colon || colon == token || colon[1] == '\0' || strchr(colon + 1, ':')) + return false; + + char major[32], minor[32]; + size_t major_len = (size_t) (colon - token); + if (major_len >= sizeof(major) || strlen(colon + 1) >= sizeof(minor)) + return false; + memcpy(major, token, major_len); + major[major_len] = '\0'; + strcpy(minor, colon + 1); + + unsigned long long value; + return parse_unsigned_token(major, 16, &value) && + parse_unsigned_token(minor, 16, &value); +} + +/* Parse the address/perms/offset/dev/inode part of a smaps header. The rest + * of the line is an optional pathname and is intentionally left opaque. */ +static bool parse_header(const char *line, smaps_vma_t *vma) +{ + const char *p = line; + errno = 0; + char *end = NULL; + unsigned long long start = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || *end != '-') + return false; + p = end + 1; + + errno = 0; + unsigned long long finish = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || start >= finish || + !isspace((unsigned char) *end)) + return false; + if (start > UINTPTR_MAX || finish > UINTPTR_MAX) + return false; + p = end; + + char token[128]; + if (!parse_token(&p, token, sizeof(token)) || strlen(token) != 4) + return false; + for (size_t i = 0; i < 3; i++) { + if (token[i] != 'r' && token[i] != 'w' && token[i] != 'x' && + token[i] != '-') + return false; + } + if (token[3] != 'p' && token[3] != 's') + return false; + memcpy(vma->perms, token, sizeof(vma->perms)); + + if (!parse_token(&p, token, sizeof(token)) || + !parse_unsigned_token(token, 16, &vma->offset)) + return false; + if (!parse_token(&p, token, sizeof(token)) || !parse_device_token(token)) + return false; + if (!parse_token(&p, token, sizeof(token))) + return false; + unsigned long long inode; + if (!parse_unsigned_token(token, 10, &inode)) + return false; + + vma->start = (uintptr_t) start; + vma->end = (uintptr_t) finish; + vma->size_kb = 0; + vma->kernel_page_kb = 0; + vma->mmu_page_kb = 0; + vma->shared_dirty_kb = 0; + vma->have_size = false; + vma->have_shared_dirty = false; + vma->have_vmflags = false; + vma->vmflags_wr = false; + return true; +} + +/* Parse a decimal field and optionally require a suffix after its number. + * Both smaps field families share the same label/whitespace/overflow rules; + * only the trailing unit differs ("kB" for the first group, none for the + * remaining numeric fields). */ +static bool parse_numeric_field(const char *line, const char *label, + const char *suffix, + unsigned long long *value) +{ + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + const char *p = skip_space(line + label_len); + const char *start = p; + while (isdigit((unsigned char) *p)) + p++; + if (p == start) + return false; + char number[64]; + size_t number_len = (size_t) (p - start); + if (number_len >= sizeof(number)) + return false; + memcpy(number, start, number_len); + number[number_len] = '\0'; + if (!parse_unsigned_token(number, 10, value)) + return false; + p = skip_space(p); + return suffix ? !strcmp(p, suffix) : *p == '\0'; +} + +static bool parse_vmflags(const char *line, bool *writable) +{ + const char *label = "VmFlags:"; + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + + const char *p = line + label_len; + bool has_wr = false; + while (*(p = skip_space(p))) { + const char *start = p; + while (*p && !isspace((unsigned char) *p)) { + if (!isalpha((unsigned char) *p)) + return false; + p++; + } + if (p == start) + return false; + if ((size_t) (p - start) == 2 && start[0] == 'w' && start[1] == 'r') + has_wr = true; + } + /* A synthetic PROT_NONE VMA has no provable access flags and therefore + * may legitimately emit an empty VmFlags field. */ + *writable = has_wr; + return true; +} + +static bool finish_vma(smaps_vma_t *vma, size_t field_index) +{ + return field_index == SMAPS_FIELD_COUNT && vma->have_size && + vma->have_shared_dirty && vma->have_vmflags; +} + +/* THPeligible and ProtectionKey are conditional in real Linux kernels. The + * elfuse provider always emits both, but qemu may legitimately omit either; + * when the parser reaches one of those labels, advance over any missing + * optional fields before parsing VmFlags. */ +static void skip_optional_fields(const char *line, size_t *field_index) +{ + if (*field_index == SMAPS_FIELD_COUNT - 3 && + (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || + !strncmp(line, "VmFlags:", 8))) + (*field_index)++; + if (*field_index == SMAPS_FIELD_COUNT - 2 && + !strncmp(line, "VmFlags:", 8)) + (*field_index)++; +} + +static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) +{ + smaps_vma_t *vmas = realloc(info->vmas, + (info->count + 1) * sizeof(*info->vmas)); + if (!vmas) + return false; + vmas[info->count++] = *vma; + info->vmas = vmas; + return true; +} + +static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) +{ + memset(info, 0, sizeof(*info)); + if (len == 0 || buf[len - 1] != '\n') + return false; /* catches a truncated final record */ + + smaps_vma_t current; + bool have_current = false; + size_t field_index = 0; + uintptr_t previous_end = 0; + char *line = buf; + while ((size_t) (line - buf) < len) { + char *next = memchr(line, '\n', len - (size_t) (line - buf)); + if (!next) + goto fail; + *next = '\0'; + + /* Linux normally places headers back-to-back; the synthetic proc + * provider separates records with one blank line. Accept that + * separator only after a complete record. */ + if (!*line) { + if (!have_current || field_index != SMAPS_FIELD_COUNT) + goto fail; + line = next + 1; + continue; + } + + smaps_vma_t header; + if (parse_header(line, &header)) { + if (have_current) { + if (!finish_vma(¤t, field_index) || + !append_vma(info, ¤t)) + goto fail; + } + if (info->count > 0 && + (header.start < previous_end || header.start <= + info->vmas[info->count - 1].start)) + goto fail; + current = header; + have_current = true; + field_index = 0; + previous_end = header.end; + } else { + if (!have_current) + goto fail; + skip_optional_fields(line, &field_index); + if (field_index >= SMAPS_FIELD_COUNT) + goto fail; + unsigned long long value; + bool writable; + if (field_index < SMAPS_KB_FIELD_COUNT && + parse_numeric_field(line, smaps_fields[field_index], "kB", + &value)) { + if (field_index == 0) + current.size_kb = value; + if (field_index == 1) + current.kernel_page_kb = value; + if (field_index == 2) + current.mmu_page_kb = value; + if (field_index == 7) + current.shared_dirty_kb = value; + if (field_index == 0) + current.have_size = true; + if (field_index == 7) + current.have_shared_dirty = true; + field_index++; + } else if (field_index >= SMAPS_KB_FIELD_COUNT && + field_index < SMAPS_FIELD_COUNT - 1 && + parse_numeric_field(line, smaps_fields[field_index], + NULL, &value)) { + field_index++; + } else if (field_index == SMAPS_FIELD_COUNT - 1 && + parse_vmflags(line, &writable)) { + current.have_vmflags = true; + current.vmflags_wr = writable; + field_index++; + } else { + goto fail; + } + } + line = next + 1; + } + + if (!have_current || !finish_vma(¤t, field_index) || + !append_vma(info, ¤t) || + info->count == 0) + goto fail; + return true; + +fail: + free(info->vmas); + memset(info, 0, sizeof(*info)); + return false; +} + +static void free_smaps(smaps_info_t *info) +{ + free(info->vmas); + memset(info, 0, sizeof(*info)); +} + +/* Read, parse, and release one smaps snapshot. The parser owns only its VMA + * array, so the transient file buffer can be cleaned up in this single place + * on both success and failure. */ +static bool load_smaps(const char *path, smaps_info_t *info) +{ + char *buf = NULL; + size_t len = 0; + bool ok = read_file_dynamic_nul(path, &buf, &len) >= 0 && + parse_smaps(buf, len, info); + free(buf); + return ok; +} + +static const smaps_vma_t *find_vma(const smaps_info_t *info, uintptr_t address) +{ + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].start <= address && address < info->vmas[i].end) + return &info->vmas[i]; + } + return NULL; +} + +static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, + uintptr_t end) +{ + size_t count = 0; + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].end > start && info->vmas[i].start < end) + count++; + } + return count; +} + +static bool validate_layout(const smaps_info_t *info, uintptr_t target, + uintptr_t stress, size_t page_size, + size_t stress_size) +{ + if (info->count <= 256) + return false; + + const smaps_vma_t *first = find_vma(info, target); + const smaps_vma_t *middle = find_vma(info, target + page_size); + const smaps_vma_t *last = find_vma(info, target + 2 * page_size); + unsigned long long page_kb = page_size / 1024; + if (!first || !middle || !last || page_kb == 0) + return false; + + if (first->start != target || first->end != target + page_size || + middle->start != target + page_size || + middle->end != target + 2 * page_size || + last->start != target + 2 * page_size || + last->end != target + 3 * page_size) + return false; + if (strcmp(first->perms, "rw-p") || strcmp(middle->perms, "r--p") || + strcmp(last->perms, "rw-p")) + return false; + if (first->size_kb != page_kb || middle->size_kb != page_kb || + last->size_kb != page_kb || !first->vmflags_wr || + middle->vmflags_wr || !last->vmflags_wr || + first->kernel_page_kb != 4 || first->mmu_page_kb != 4 || + middle->kernel_page_kb != 4 || middle->mmu_page_kb != 4 || + last->kernel_page_kb != 4 || last->mmu_page_kb != 4) + return false; + + /* Every stress-map page alternates permissions, so each page must remain + * its own VMA. Requiring the full count makes a short read or a producer + * cap observable instead of merely checking that some blocks exceed 256. + */ + size_t expected_stress_vmas = stress_size / page_size; + if (expected_stress_vmas <= 256 || + count_vmas_in_range(info, stress, stress + stress_size) != + expected_stress_vmas) + return false; + for (size_t i = 0; i < expected_stress_vmas; i++) { + const smaps_vma_t *page = find_vma(info, stress + i * page_size); + if (!page || page->start != stress + i * page_size || + page->end != stress + (i + 1) * page_size) + return false; + } + return true; +} + +static bool read_exact(int fd, void *data, size_t len) +{ + char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = read(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + +static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, + size_t stress_size) +{ + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + if (!load_smaps(paths[i], &info)) + return 1; + bool ok = validate_layout(&info, target, stress, page_size, + stress_size); + const smaps_vma_t *first = find_vma(&info, target); + const smaps_vma_t *middle = find_vma(&info, target + page_size); + const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); + const smaps_vma_t *stress_ro = find_vma(&info, stress); + const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || + first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || + last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || + stress_rw->shared_dirty_kb == 0) { + free_smaps(&info); + return 1; + } + free_smaps(&info); + } + return 0; +} + +int main(void) +{ + long page_size_long = sysconf(_SC_PAGESIZE); + TEST("smaps page size and VMA fixture"); + if (page_size_long < 1024 || page_size_long % 1024 != 0) { + FAIL("sysconf(_SC_PAGESIZE)"); + SUMMARY("test-proc-smap"); + return 1; + } + PASS(); + size_t page_size = (size_t) page_size_long; + + size_t target_size = 3 * page_size; + char *target = mmap(NULL, target_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t stress_pages = 320; + size_t stress_size = stress_pages * page_size; + char *stress = MAP_FAILED; + bool fixture_ok = target != MAP_FAILED; + if (fixture_ok) + stress = mmap(NULL, stress_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fixture_ok = fixture_ok && stress != MAP_FAILED; + if (fixture_ok) { + *(volatile unsigned char *) target = 0x5a; /* make one page dirty */ + fixture_ok = mprotect(target + page_size, page_size, PROT_READ) == 0; + } + if (fixture_ok) { + /* Start with a read-only page so an allocator placing this mapping + * immediately after target cannot merge target's final rw page into + * the stress range. Alternating permissions keeps every stress page + * as a distinct VMA while preserving the >256 completeness probe. */ + for (size_t i = 0; i < stress_pages; i += 2) { + if (mprotect(stress + i * page_size, page_size, PROT_READ) < 0) { + fixture_ok = false; + break; + } + } + } + + TEST("smaps headers, order, fields, and completeness"); + if (!fixture_ok) { + FAIL("mmap/mprotect fixture"); + } else { + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + bool ok = true; + size_t expected_count = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + bool parsed = load_smaps(paths[i], &info); + if (!parsed || + !validate_layout(&info, (uintptr_t) target, + (uintptr_t) stress, page_size, stress_size)) { + ok = false; + } else if (i == 0) { + expected_count = info.count; + } else if (info.count != expected_count) { + ok = false; + } + if (parsed) + free_smaps(&info); + } + EXPECT_TRUE(ok, "smaps parser/layout/completeness"); + } + + TEST("fork Shared_Dirty positive and read-only negative"); + if (!fixture_ok) { + FAIL("fixture unavailable"); + } else { + int pipefd[2] = {-1, -1}; + pid_t pid = pipe(pipefd) == 0 ? fork() : -1; + if (pid < 0) { + FAIL("pipe/fork"); + if (pipefd[0] >= 0) + close(pipefd[0]); + if (pipefd[1] >= 0) + close(pipefd[1]); + } else if (pid == 0) { + close(pipefd[0]); + int result = child_probe((uintptr_t) target, (uintptr_t) stress, + page_size, stress_size); + (void) write_fd_all(pipefd[1], &result, sizeof(result)); + close(pipefd[1]); + _exit(result); + } else { + close(pipefd[1]); + int result = 1; + bool received = read_exact(pipefd[0], &result, sizeof(result)); + close(pipefd[0]); + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + EXPECT_TRUE(received && waited && result == 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0, + "fork Shared_Dirty accounting"); + } + } + + if (stress != MAP_FAILED) + munmap(stress, stress_size); + if (target != MAP_FAILED) + munmap(target, target_size); + SUMMARY("test-proc-smap"); + return fails == 0 ? 0 : 1; +} diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c new file mode 100644 index 00000000..e926bb1b --- /dev/null +++ b/tests/test-string-builder-host.c @@ -0,0 +1,182 @@ +/* + * Native-host unit tests for string_builder_t. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "string-builder.h" + +static void expect_string(const string_builder_t *builder, const char *expected) +{ + size_t length = strlen(expected); + assert(string_builder_length(builder) == length); + assert(builder->storage.raw.count == length); + if (string_builder_data_const(builder) != NULL) { + assert(strcmp(string_builder_data_const(builder), expected) == 0); + assert(string_builder_data_const(builder)[length] == '\0'); + } else { + assert(length == 0); + } +} + +static void test_zero_and_initial_capacity(void) +{ + string_builder_t zero = {0}; + assert(string_builder_data(&zero) == NULL); + assert(string_builder_length(&zero) == 0); + assert(string_builder_capacity(&zero) == 0); + /* The public API also accepts a plain {0} value without an init call. */ + assert(string_builder_append(&zero, "zero") == 0); + expect_string(&zero, "zero"); + string_builder_destroy(&zero); + + string_builder_t lazy = {0}; + assert(string_builder_init(&lazy, 0) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_capacity(&lazy) == 0); + assert(string_builder_data(&lazy) == NULL); + assert(string_builder_reserve(&lazy, 4) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_data(&lazy)[0] == '\0'); + assert(string_builder_append(&lazy, "lazy") == 0); + assert(string_builder_capacity(&lazy) >= string_builder_length(&lazy) + 1); + expect_string(&lazy, "lazy"); + string_builder_destroy(&lazy); + + string_builder_t initial = {0}; + /* initial_capacity includes the trailing NUL byte. */ + assert(string_builder_init(&initial, 32) == 0); + assert(string_builder_capacity(&initial) >= 32); + assert(string_builder_data(&initial) != NULL); + assert(string_builder_length(&initial) == 0); + expect_string(&initial, ""); + string_builder_destroy(&initial); +} + +static void test_text_and_formatted_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + assert(string_builder_append(&builder, "prefix") == 0); + + assert(string_builder_appendf(&builder, ":%s:%d", "formatted", 42) == 0); + expect_string(&builder, "prefix:formatted:42"); + + /* An empty C string is a no-op. */ + size_t old_length = string_builder_length(&builder); + assert(string_builder_append(&builder, "") == 0); + assert(string_builder_length(&builder) == old_length); + expect_string(&builder, "prefix:formatted:42"); + string_builder_destroy(&builder); +} + +static void test_c_string_semantics(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "prefix") == 0); + + const char embedded[] = {'a', '\0', 'b', '\0'}; + errno = 0; + assert(string_builder_append(&builder, embedded) == 0); + assert(errno == 0); + expect_string(&builder, "prefixa"); + + errno = 0; + assert(string_builder_appendf(&builder, "x%c y", '\0') == 0); + assert(errno == 0); + expect_string(&builder, "prefixax"); + string_builder_destroy(&builder); + + /* A formatted NUL also terminates the appended C-string prefix when the + * first sizing pass has room. + */ + string_builder_t fit = {0}; + assert(string_builder_init(&fit, 16) == 0); + errno = 0; + assert(string_builder_appendf(&fit, "a%c%d", '\0', 1) == 0); + assert(errno == 0); + expect_string(&fit, "a"); + string_builder_destroy(&fit); +} + +static void test_growth_preserves_content(void) +{ + enum { COUNT = 4096 }; + char expected[COUNT]; + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + for (size_t i = 0; i < COUNT; i++) { + expected[i] = (char)('A' + (i % 26)); + char chunk[2] = {expected[i], '\0'}; + assert(string_builder_append(&builder, chunk) == 0); + } + assert(string_builder_length(&builder) == COUNT); + assert(memcmp(string_builder_data_const(&builder), expected, COUNT) == 0); + assert(string_builder_data_const(&builder)[COUNT] == '\0'); + assert(string_builder_capacity(&builder) >= COUNT + 1); + string_builder_destroy(&builder); +} + +static void test_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 4) == 0); + assert(string_builder_append(&builder, "abc") == 0); + const char *alias = string_builder_data_const(&builder) + 1; + assert(string_builder_append(&builder, alias) == 0); + const char expected[] = "abcbc"; + expect_string(&builder, expected); + string_builder_destroy(&builder); +} + +static void test_overflow_preserves_content(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 0) == 0); + assert(string_builder_appendf(&builder, "prefix:%d", 7) == 0); + + char snapshot[32]; + assert(string_builder_length(&builder) < sizeof(snapshot)); + memcpy(snapshot, string_builder_data_const(&builder), string_builder_length(&builder)); + size_t old_length = string_builder_length(&builder); + size_t old_capacity = string_builder_capacity(&builder); + + errno = 0; + assert(string_builder_reserve(&builder, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + + errno = 0; + assert(string_builder_append(&builder, NULL) == -1); + assert(errno == EILSEQ); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + string_builder_destroy(&builder); +} + +int main(void) +{ + test_zero_and_initial_capacity(); + test_text_and_formatted_append(); + test_c_string_semantics(); + test_growth_preserves_content(); + test_alias_append(); + test_overflow_preserves_content(); + puts("test-string-builder-host: PASS"); + return 0; +} diff --git a/tests/test-util.h b/tests/test-util.h index 5a1a1a71..d3aa1c6f 100644 --- a/tests/test-util.h +++ b/tests/test-util.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,69 @@ static inline ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) return total; } +/* Read a file whose size is not available from st_size (for example a proc + * file) until EOF, growing the buffer and appending a NUL terminator. */ +static inline ssize_t read_file_dynamic_nul(const char *path, char **buf_out, + size_t *len_out) +{ + if (!path || !buf_out || !len_out) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + size_t cap = 64 * 1024; + size_t len = 0; + char *buf = malloc(cap); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + + for (;;) { + if (len + 1 >= cap) { + if (cap > SIZE_MAX / 2) { + free(buf); + close(fd); + errno = EOVERFLOW; + return -1; + } + size_t new_cap = cap * 2; + char *new_buf = realloc(buf, new_cap); + if (!new_buf) { + free(buf); + close(fd); + errno = ENOMEM; + return -1; + } + buf = new_buf; + cap = new_cap; + } + + ssize_t n = read(fd, buf + len, cap - len - 1); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + free(buf); + close(fd); + return -1; + } + if (n == 0) + break; + len += (size_t) n; + } + close(fd); + + buf[len] = '\0'; + *buf_out = buf; + *len_out = len; + return (ssize_t) len; +} + static inline ssize_t raw_read_fd_all_nul(int fd, char *buf, size_t bufsz) { if (bufsz == 0) From de34fdcdfe3891d4418dfd9fc9bc4f2295f6af85 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:16:30 +0800 Subject: [PATCH 02/14] Fix CI analysis and smaps matrix Keep static-analysis checks and the smaps test matrix green after adding the new procfs emulation coverage. --- src/dynamic-array.c | 40 ++++++++++------- src/dynamic-array.h | 56 ++++++++++++------------ src/runtime/procemu.c | 65 ++++++++++++++-------------- src/string-builder.c | 54 +++++++++++++---------- src/string-builder.h | 3 +- tests/test-dynamic-array-host.c | 27 +++++++++++- tests/test-matrix.sh | 15 ++++--- tests/test-proc-smap.c | 74 +++++++++++++++++--------------- tests/test-string-builder-host.c | 11 +++-- tests/test-util.h | 3 +- 10 files changed, 205 insertions(+), 143 deletions(-) diff --git a/src/dynamic-array.c b/src/dynamic-array.c index a4dd63c8..aad9dda4 100644 --- a/src/dynamic-array.c +++ b/src/dynamic-array.c @@ -12,7 +12,7 @@ #include #include -#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t)8) +#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t) 8) /* Set errno for a bad argument and return a standard failure code. */ static int dynamic_array_invalid(void) @@ -31,7 +31,8 @@ static int dynamic_array_validate(const dynamic_array_t *array) /* Return count + extra after guarding against size_t overflow. */ static int dynamic_array_count_plus(const dynamic_array_t *array, - size_t extra, size_t *total) + size_t extra, + size_t *total) { if (extra > SIZE_MAX - array->count) { errno = EOVERFLOW; @@ -42,7 +43,8 @@ static int dynamic_array_count_plus(const dynamic_array_t *array, } /* Compute the byte size for a number of elements with overflow protection. */ -static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, +static int dynamic_array_bytes(const dynamic_array_t *array, + size_t count, size_t *bytes) { if (count != 0 && array->element_size > SIZE_MAX / count) { @@ -55,7 +57,8 @@ static int dynamic_array_bytes(const dynamic_array_t *array, size_t count, /* Return the offset of an in-array source span, including capacity bytes. */ static int dynamic_array_source_offset(const dynamic_array_t *array, - const void *source, size_t bytes, + const void *source, + size_t bytes, size_t *offset) { if (array->data == NULL || source == NULL) @@ -66,7 +69,7 @@ static int dynamic_array_source_offset(const dynamic_array_t *array, if (address < base) return 0; uintptr_t delta = address - base; - if (delta > (uintptr_t)SIZE_MAX) + if (delta > (uintptr_t) SIZE_MAX) return 0; size_t start = (size_t) delta; size_t allocation_bytes; @@ -135,7 +138,7 @@ void dynamic_array_destroy(dynamic_array_t *array) if (array == NULL) return; free(array->data); - *array = (dynamic_array_t){0}; + *array = (dynamic_array_t) {0}; } /* Ensure the array has capacity for at least extra additional elements. */ @@ -186,8 +189,10 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count) } if (count > array->count) { size_t old_bytes, new_bytes; - (void) dynamic_array_bytes(array, array->count, &old_bytes); - (void) dynamic_array_bytes(array, count, &new_bytes); + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; + if (dynamic_array_bytes(array, count, &new_bytes) < 0) + return -1; memset((unsigned char *) array->data + old_bytes, 0, new_bytes - old_bytes); } @@ -196,7 +201,8 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count) } /* Append multiple elements from source memory to the end of the array. */ -int dynamic_array_append_n(dynamic_array_t *array, const void *data, +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, size_t count) { if (dynamic_array_validate(array) < 0) @@ -220,15 +226,18 @@ int dynamic_array_append_n(dynamic_array_t *array, const void *data, if (aliases) data = (const unsigned char *) array->data + offset; size_t old_bytes; - (void) dynamic_array_bytes(array, array->count, &old_bytes); + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; memmove((unsigned char *) array->data + old_bytes, data, bytes); array->count = total; return 0; } /* Insert multiple elements at index while preserving existing elements. */ -int dynamic_array_insert_n(dynamic_array_t *array, size_t index, - const void *data, size_t count) +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count) { if (dynamic_array_validate(array) < 0) return -1; @@ -252,8 +261,8 @@ int dynamic_array_insert_n(dynamic_array_t *array, size_t index, return -1; size_t source_offset = 0; - int aliases = dynamic_array_source_offset(array, data, bytes, - &source_offset); + int aliases = + dynamic_array_source_offset(array, data, bytes, &source_offset); void *temporary = NULL; if (aliases) { temporary = malloc(bytes); @@ -285,7 +294,8 @@ int dynamic_array_append_one(dynamic_array_t *array, const void *data) } /* Insert a single element by forwarding to insert_n. */ -int dynamic_array_insert_one(dynamic_array_t *array, size_t index, +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, const void *data) { return dynamic_array_insert_n(array, index, data, 1); diff --git a/src/dynamic-array.h b/src/dynamic-array.h index 7e95c592..3899194a 100644 --- a/src/dynamic-array.h +++ b/src/dynamic-array.h @@ -63,24 +63,30 @@ int dynamic_array_resize(dynamic_array_t *array, size_t count); /* Append one element, or count elements when the three-argument form is * used. The macro keeps both forms available to callers. */ int dynamic_array_append_one(dynamic_array_t *array, const void *data); -int dynamic_array_append_n(dynamic_array_t *array, const void *data, +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, size_t count); #define DYNAMIC_ARRAY_APPEND_PICK(_1, _2, _3, NAME, ...) NAME -#define dynamic_array_append(...) \ - DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ - dynamic_array_append_one, dynamic_array_append_dummy) \ - (__VA_ARGS__) +#define dynamic_array_append(...) \ + DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ + dynamic_array_append_one, \ + dynamic_array_append_dummy) \ + (__VA_ARGS__) /* Insert one element, or count elements in the four-argument form. */ -int dynamic_array_insert_one(dynamic_array_t *array, size_t index, +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, const void *data); -int dynamic_array_insert_n(dynamic_array_t *array, size_t index, - const void *data, size_t count); +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count); #define DYNAMIC_ARRAY_INSERT_PICK(_1, _2, _3, _4, NAME, ...) NAME -#define dynamic_array_insert(...) \ - DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ - dynamic_array_insert_one, dynamic_array_insert_dummy) \ - (__VA_ARGS__) +#define dynamic_array_insert(...) \ + DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ + dynamic_array_insert_one, \ + dynamic_array_insert_dummy) \ + (__VA_ARGS__) /* Return an element pointer, or NULL with errno=EINVAL for a bad index. */ void *dynamic_array_at(dynamic_array_t *array, size_t index); @@ -101,8 +107,8 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); sizeof(type)); \ } \ /* Initialize typed array and preallocate initial slots. */ \ - DYNAMIC_ARRAY_INLINE int name##_init_with_capacity(name##_t *array, \ - size_t initial_capacity) \ + DYNAMIC_ARRAY_INLINE int name##_init_with_capacity( \ + name##_t *array, size_t initial_capacity) \ { \ return dynamic_array_init_with_capacity( \ array != NULL ? &array->raw : NULL, sizeof(type), \ @@ -156,8 +162,7 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); } \ /* Append a typed range of values. */ \ DYNAMIC_ARRAY_INLINE int name##_append_n(name##_t *array, \ - const type *values, \ - size_t count) \ + const type *values, size_t count) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -167,9 +172,8 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); count)); \ } \ /* Insert one typed value through a pointer form. */ \ - DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, \ - size_t index, \ - const type *value) \ + DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, size_t index, \ + const type *value) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -180,21 +184,19 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); } \ /* Insert one typed value by value. */ \ DYNAMIC_ARRAY_INLINE int name##_insert_value(name##_t *array, \ - size_t index, \ - type value) \ + size_t index, type value) \ { \ return name##_insert_ptr(array, index, &value); \ } \ /* Insert one typed value from a pointer argument. */ \ DYNAMIC_ARRAY_INLINE int name##_insert(name##_t *array, size_t index, \ - const type *value) \ + const type *value) \ { \ return name##_insert_ptr(array, index, value); \ } \ /* Insert a typed range of values at index. */ \ - DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, \ - size_t index, const type *values, \ - size_t count) \ + DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, size_t index, \ + const type *values, size_t count) \ { \ int was_uninitialized = dynamic_array_typed_prepare( \ array != NULL ? &array->raw : NULL, sizeof(type)); \ @@ -207,11 +209,11 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); DYNAMIC_ARRAY_INLINE type *name##_at(name##_t *array, size_t index) \ { \ return (type *) dynamic_array_at(array != NULL ? &array->raw : NULL, \ - index); \ + index); \ } \ /* Return a typed const pointer to the element at index. */ \ DYNAMIC_ARRAY_INLINE const type *name##_at_const(const name##_t *array, \ - size_t index) \ + size_t index) \ { \ return (const type *) dynamic_array_at_const( \ array != NULL ? &array->raw : NULL, index); \ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index fb6f3681..2359ea56 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -167,7 +167,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) if (out != i) *maps_entries_at(entries, out) = *current; } - (void)maps_entries_resize(entries, out + 1); + (void) maps_entries_resize(entries, out + 1); } /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily @@ -467,8 +467,8 @@ static void proc_tmpdir_cleanup(void) /* Remove known files inside // and / */ char path[256]; - const char *files[] = {"stat", "status", "cmdline", "maps", "smaps", - "exe", NULL}; + const char *files[] = {"stat", "status", "cmdline", "maps", + "smaps", "exe", NULL}; char piddir[160]; /* Reconstruct pid subdir by scanning for the first numeric entry */ @@ -2365,7 +2365,7 @@ static int pty_open_master(int linux_flags) * record and output generation does not block page-table mutations. */ static int proc_build_maps_entries(const guest_t *g, - maps_entries_t *entries_out) + maps_entries_t *entries_out) { if (!g || !entries_out) { errno = EINVAL; @@ -2439,13 +2439,12 @@ static int proc_build_maps_entries(const guest_t *g, continue; if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, - r->prot, r->flags, r->offset, - r->name) < 0) + (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, + r->flags, r->offset, r->name) < 0) goto out_unlock; } maps_entries_merge_adjacent(&entries); - result = (int)maps_entries_count(&entries); + result = (int) maps_entries_count(&entries); out_unlock: pthread_mutex_unlock(&mmap_lock); @@ -2455,7 +2454,7 @@ static int proc_build_maps_entries(const guest_t *g, return -1; } *entries_out = entries; - return (int)maps_entries_count(&entries); + return (int) maps_entries_count(&entries); } /* Format the common VMA header. The maps and smaps header must stay byte-for- @@ -2472,10 +2471,10 @@ static int proc_format_maps_header(const maps_entry_t *e, perms[3] = (e->flags & LINUX_MAP_SHARED) ? 's' : 'p'; perms[4] = '\0'; - int header_len = snprintf( - header, headersz, "%llx-%llx %s %08llx 00:00 0", - (unsigned long long) e->start, (unsigned long long) e->end, perms, - (unsigned long long) e->offset); + int header_len = + snprintf(header, headersz, "%llx-%llx %s %08llx 00:00 0", + (unsigned long long) e->start, (unsigned long long) e->end, + perms, (unsigned long long) e->offset); if (header_len < 0) return -1; if ((size_t) header_len >= headersz) @@ -2531,7 +2530,7 @@ static int proc_open_self_maps(const guest_t *g) /* Emit lines after merging so buffer accounting is centralized. */ for (int i = 0; i < nentries; i++) { - const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char line[256]; int line_len = proc_format_maps_header(e, line, sizeof(line)); if (line_len < 0 || @@ -2539,10 +2538,12 @@ static int proc_open_self_maps(const guest_t *g) goto out; } - log_debug("/proc/self/maps (%zu bytes):\n%.*s", string_builder_length(&builder), - (int)string_builder_length(&builder), - string_builder_data_const(&builder) ? string_builder_data_const(&builder) - : ""); + log_debug("/proc/self/maps (%zu bytes):\n%.*s", + string_builder_length(&builder), + (int) string_builder_length(&builder), + string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : ""); result = proc_synthetic_fd(string_builder_data_const(&builder) ? string_builder_data_const(&builder) : "", @@ -2573,7 +2574,7 @@ static int proc_open_self_smaps(const guest_t *g) bool fork_child = proc_get_ppid() != 0; for (int i = 0; i < nentries; i++) { - const maps_entry_t *e = maps_entries_at_const(&entries, (size_t)i); + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char header[256]; int header_len = proc_format_maps_header(e, header, sizeof(header)); if (header_len < 0) @@ -2583,23 +2584,23 @@ static int proc_open_self_smaps(const guest_t *g) bool anonymous = (e->flags & LINUX_MAP_ANONYMOUS) != 0; bool shared = (e->flags & LINUX_MAP_SHARED) != 0; bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; - bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && - anonymous && !shared; - bool logical_shared_dirty = fork_child && private_anon && - (e->prot & LINUX_PROT_WRITE); + bool private_anon = + (e->flags & LINUX_MAP_PRIVATE) && anonymous && !shared; + bool logical_shared_dirty = + fork_child && private_anon && (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; char vmflags[64]; size_t vmflags_len = 0; -#define APPEND_VMFLAG(flag) \ - do { \ - const char *token = (flag); \ - size_t token_len = strlen(token); \ - if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ - vmflags[vmflags_len++] = ' '; \ - memcpy(vmflags + vmflags_len, token, token_len); \ - vmflags_len += token_len; \ - } \ +#define APPEND_VMFLAG(flag) \ + do { \ + const char *token = (flag); \ + size_t token_len = strlen(token); \ + if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ + vmflags[vmflags_len++] = ' '; \ + memcpy(vmflags + vmflags_len, token, token_len); \ + vmflags_len += token_len; \ + } \ } while (0) /* Only flags directly evidenced by the tracked VMA are reported. In * particular, do not invent Linux max-permission/accounting flags diff --git a/src/string-builder.c b/src/string-builder.c index 5cd994d2..fa134065 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -30,7 +30,7 @@ int string_builder_init(string_builder_t *builder, size_t initial_capacity) return 0; } if (string_builder_storage_init_with_capacity(&builder->storage, - initial_capacity) < 0) + initial_capacity) < 0) return -1; string_builder_data(builder)[0] = '\0'; return 0; @@ -61,15 +61,16 @@ int string_builder_reserve(string_builder_t *builder, size_t extra) } if (string_builder_storage_reserve(&builder->storage, extra + 1) < 0) return -1; - string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = - '\0'; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; return 0; } /* Locate a source span that aliases the builder allocation. The offset must be * captured before reserve because reserve may move the allocation. */ static int string_builder_source_offset(const string_builder_t *builder, - const void *source, size_t length, + const void *source, + size_t length, size_t *offset) { const char *base_ptr = string_builder_data_const(builder); @@ -92,12 +93,13 @@ static int string_builder_source_offset(const string_builder_t *builder, /* Commit string bytes through the generic array and restore the terminator. */ static int string_builder_commit_append(string_builder_t *builder, - const char *data, size_t len) + const char *data, + size_t len) { if (string_builder_storage_append_n(&builder->storage, data, len) < 0) return -1; - string_builder_data(builder)[string_builder_storage_count(&builder->storage)] = - '\0'; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; return 0; } @@ -112,8 +114,8 @@ int string_builder_append(string_builder_t *builder, const char *text) return 0; size_t source_offset = 0; - int aliases = string_builder_source_offset(builder, text, len, - &source_offset); + int aliases = + string_builder_source_offset(builder, text, len, &source_offset); if (string_builder_reserve(builder, len) < 0) return -1; if (aliases) @@ -130,8 +132,10 @@ static int string_builder_format_failure(void) } /* Call vsnprintf with a copied argument list for the current sizing pass. */ -static int string_builder_vsnprintf(char *destination, size_t available, - const char *format, va_list arguments) +static int string_builder_vsnprintf(char *destination, + size_t available, + const char *format, + va_list arguments) { va_list pass; va_copy(pass, arguments); @@ -167,15 +171,17 @@ static int string_builder_prepare_appendf(string_builder_t *builder, } /* Format into the current tail and report whether the result was truncated. */ -static string_builder_format_status_t -string_builder_try_format(string_builder_t *builder, size_t old_len, - const char *format, va_list arguments, - size_t *written) +static string_builder_format_status_t string_builder_try_format( + string_builder_t *builder, + size_t old_len, + const char *format, + va_list arguments, + size_t *written) { char *data = string_builder_data(builder); size_t available = string_builder_capacity(builder) - old_len; - int formatted = string_builder_vsnprintf(data + old_len, available, format, - arguments); + int formatted = + string_builder_vsnprintf(data + old_len, available, format, arguments); if (formatted < 0) return STRING_BUILDER_FORMAT_ERROR; @@ -260,24 +266,28 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) /* Return mutable storage for the builder, if it has been allocated. */ char *string_builder_data(string_builder_t *builder) { - return builder != NULL ? string_builder_storage_data(&builder->storage) : NULL; + return builder != NULL ? string_builder_storage_data(&builder->storage) + : NULL; } /* Return const storage for the builder, if it has been allocated. */ const char *string_builder_data_const(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_data_const(&builder->storage) - : NULL; + return builder != NULL + ? string_builder_storage_data_const(&builder->storage) + : NULL; } /* Return the number of data bytes currently stored. */ size_t string_builder_length(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_count(&builder->storage) : 0; + return builder != NULL ? string_builder_storage_count(&builder->storage) + : 0; } /* Return allocated capacity in bytes, including the terminating NUL. */ size_t string_builder_capacity(const string_builder_t *builder) { - return builder != NULL ? string_builder_storage_capacity(&builder->storage) : 0; + return builder != NULL ? string_builder_storage_capacity(&builder->storage) + : 0; } diff --git a/src/string-builder.h b/src/string-builder.h index 8feb6ae3..f9fb1c0c 100644 --- a/src/string-builder.h +++ b/src/string-builder.h @@ -52,7 +52,8 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...); */ char *string_builder_data(string_builder_t *builder); -/* Return read-only builder data, or NULL when builder is NULL or unallocated. */ +/* Return read-only builder data, or NULL when builder is NULL or unallocated. + */ const char *string_builder_data_const(const string_builder_t *builder); /* Return the number of data bytes currently stored, excluding the NUL. */ diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c index 73938842..6a3ea3ce 100644 --- a/tests/test-dynamic-array-host.c +++ b/tests/test-dynamic-array-host.c @@ -65,7 +65,8 @@ static void test_alias_and_resize(void) int *old_data = int_array_data(&values); int *alias = int_array_at(&values, 1); assert(int_array_append(&values, alias) == 0); - assert(int_array_data(&values) != old_data || int_array_capacity(&values) > 4); + assert(int_array_data(&values) != old_data || + int_array_capacity(&values) > 4); assert(int_array_count(&values) == 5); assert(*int_array_at(&values, 4) == 2); assert(int_array_resize(&values, 8) == 0); @@ -96,6 +97,30 @@ static void test_invalid_and_overflow(void) assert(errno == EOVERFLOW); assert(raw.data == NULL && raw.count == 0 && raw.capacity == 0); dynamic_array_destroy(&raw); + + /* A malformed metadata state must not let failed byte-size calculations + * feed uninitialized offsets into memory operations. */ + dynamic_array_t resize_overflow = { + .data = NULL, + .count = 0, + .capacity = 2, + .element_size = SIZE_MAX, + }; + errno = 0; + assert(dynamic_array_resize(&resize_overflow, 2) == -1); + assert(errno == EOVERFLOW); + assert(resize_overflow.count == 0); + + dynamic_array_t append_overflow = { + .data = NULL, + .count = SIZE_MAX / 2 + 1, + .capacity = SIZE_MAX, + .element_size = 2, + }; + errno = 0; + assert(dynamic_array_append(&append_overflow, &value) == -1); + assert(errno == EOVERFLOW); + assert(append_overflow.count == SIZE_MAX / 2 + 1); } int main(void) diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index a5cbdb11..3c9eaaf4 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -243,10 +243,10 @@ unstage_sysroot_fixtures() # Generic test helpers. -# The qemu reference lane now runs every matrix test against the real Alpine -# linux-virt kernel, so QEMU_SKIP is empty. Add a test's name here only if it -# asserts elfuse-specific behavior a real kernel does not honor; it still runs -# in elfuse-aarch64 mode and in 'make check'. +# The qemu reference lane runs the portable matrix tests against the real +# Alpine linux-virt kernel. Add a test's name here only if it asserts +# elfuse-specific behavior a real kernel does not honor; it still runs in +# elfuse-aarch64 mode and in 'make check'. # # The two oom_adj/oom_score_adj sendfile-and-copy_file_range-interception # subtests that used to make test-io-opt diverge here were split out into @@ -283,6 +283,7 @@ QEMU_SKIP=" test-fd-family test-scm-creds test-proc-fidelity + test-proc-smap " # test-session: getpgid/getsid/setsid assume the test is its own session and # process-group leader, true when elfuse launches it directly but not when @@ -352,6 +353,10 @@ QEMU_SKIP=" # under elfuse, while a real kernel allows the open and only rejects the # write -- a genuine behavioral difference worth reviewing on its own, # not just an environment artifact. +# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including a +# coarse Shared_Dirty compatibility signal for untouched writable VMAs. Real +# Linux smaps exposes kernel-owned VMA/page accounting instead, so this is +# intentionally not a reference-kernel invariant. is_qemu_skipped() { @@ -1233,7 +1238,7 @@ run_suite() # detector does not recognize yet. EXPECTED_BASELINES=( "elfuse-aarch64|239|0" - "qemu-aarch64|219|0" + "qemu-aarch64|218|0" "elfuse-x86_64:apple-m1-m2|71|0" "elfuse-x86_64:apple-m3-plus|71|0" "elfuse-x86_64:apple-unknown|71|0" diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 99786148..97a02e77 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -39,15 +39,13 @@ typedef struct { } smaps_vma_t; static const char *const smaps_fields[] = { - "Size:", "KernelPageSize:", "MMUPageSize:", - "Rss:", "Pss:", "Pss_Dirty:", - "Shared_Clean:", "Shared_Dirty:", "Private_Clean:", - "Private_Dirty:", "Referenced:", "Anonymous:", - "KSM:", "LazyFree:", "AnonHugePages:", - "ShmemPmdMapped:", - "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", - "Swap:", "SwapPss:", "Locked:", - "THPeligible:", "ProtectionKey:", "VmFlags:", + "Size:", "KernelPageSize:", "MMUPageSize:", "Rss:", + "Pss:", "Pss_Dirty:", "Shared_Clean:", "Shared_Dirty:", + "Private_Clean:", "Private_Dirty:", "Referenced:", "Anonymous:", + "KSM:", "LazyFree:", "AnonHugePages:", "ShmemPmdMapped:", + "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", "Swap:", + "SwapPss:", "Locked:", "THPeligible:", "ProtectionKey:", + "VmFlags:", }; #define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) @@ -80,7 +78,8 @@ static bool parse_token(const char **cursor, char *token, size_t token_size) return true; } -static bool parse_unsigned_token(const char *token, int base, +static bool parse_unsigned_token(const char *token, + int base, unsigned long long *value) { if (!token[0]) @@ -179,7 +178,8 @@ static bool parse_header(const char *line, smaps_vma_t *vma) * Both smaps field families share the same label/whitespace/overflow rules; * only the trailing unit differs ("kB" for the first group, none for the * remaining numeric fields). */ -static bool parse_numeric_field(const char *line, const char *label, +static bool parse_numeric_field(const char *line, + const char *label, const char *suffix, unsigned long long *value) { @@ -247,15 +247,14 @@ static void skip_optional_fields(const char *line, size_t *field_index) (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || !strncmp(line, "VmFlags:", 8))) (*field_index)++; - if (*field_index == SMAPS_FIELD_COUNT - 2 && - !strncmp(line, "VmFlags:", 8)) + if (*field_index == SMAPS_FIELD_COUNT - 2 && !strncmp(line, "VmFlags:", 8)) (*field_index)++; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) { - smaps_vma_t *vmas = realloc(info->vmas, - (info->count + 1) * sizeof(*info->vmas)); + smaps_vma_t *vmas = + realloc(info->vmas, (info->count + 1) * sizeof(*info->vmas)); if (!vmas) return false; vmas[info->count++] = *vma; @@ -298,8 +297,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) goto fail; } if (info->count > 0 && - (header.start < previous_end || header.start <= - info->vmas[info->count - 1].start)) + (header.start < previous_end || + header.start <= info->vmas[info->count - 1].start)) goto fail; current = header; have_current = true; @@ -347,8 +346,7 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) } if (!have_current || !finish_vma(¤t, field_index) || - !append_vma(info, ¤t) || - info->count == 0) + !append_vma(info, ¤t) || info->count == 0) goto fail; return true; @@ -386,7 +384,8 @@ static const smaps_vma_t *find_vma(const smaps_info_t *info, uintptr_t address) return NULL; } -static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, +static size_t count_vmas_in_range(const smaps_info_t *info, + uintptr_t start, uintptr_t end) { size_t count = 0; @@ -397,8 +396,10 @@ static size_t count_vmas_in_range(const smaps_info_t *info, uintptr_t start, return count; } -static bool validate_layout(const smaps_info_t *info, uintptr_t target, - uintptr_t stress, size_t page_size, +static bool validate_layout(const smaps_info_t *info, + uintptr_t target, + uintptr_t stress, + size_t page_size, size_t stress_size) { if (info->count <= 256) @@ -421,11 +422,11 @@ static bool validate_layout(const smaps_info_t *info, uintptr_t target, strcmp(last->perms, "rw-p")) return false; if (first->size_kb != page_kb || middle->size_kb != page_kb || - last->size_kb != page_kb || !first->vmflags_wr || - middle->vmflags_wr || !last->vmflags_wr || - first->kernel_page_kb != 4 || first->mmu_page_kb != 4 || - middle->kernel_page_kb != 4 || middle->mmu_page_kb != 4 || - last->kernel_page_kb != 4 || last->mmu_page_kb != 4) + last->size_kb != page_kb || !first->vmflags_wr || middle->vmflags_wr || + !last->vmflags_wr || first->kernel_page_kb != 4 || + first->mmu_page_kb != 4 || middle->kernel_page_kb != 4 || + middle->mmu_page_kb != 4 || last->kernel_page_kb != 4 || + last->mmu_page_kb != 4) return false; /* Every stress-map page alternates permissions, so each page must remain @@ -461,7 +462,9 @@ static bool read_exact(int fd, void *data, size_t len) return true; } -static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, +static int child_probe(uintptr_t target, + uintptr_t stress, + size_t page_size, size_t stress_size) { char pid_path[64]; @@ -472,8 +475,8 @@ static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, smaps_info_t info; if (!load_smaps(paths[i], &info)) return 1; - bool ok = validate_layout(&info, target, stress, page_size, - stress_size); + bool ok = + validate_layout(&info, target, stress, page_size, stress_size); const smaps_vma_t *first = find_vma(&info, target); const smaps_vma_t *middle = find_vma(&info, target + page_size); const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); @@ -536,7 +539,8 @@ int main(void) FAIL("mmap/mprotect fixture"); } else { char pid_path[64]; - snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", + (long) getpid()); const char *paths[] = {"/proc/self/smaps", pid_path}; bool ok = true; size_t expected_count = 0; @@ -544,8 +548,8 @@ int main(void) smaps_info_t info; bool parsed = load_smaps(paths[i], &info); if (!parsed || - !validate_layout(&info, (uintptr_t) target, - (uintptr_t) stress, page_size, stress_size)) { + !validate_layout(&info, (uintptr_t) target, (uintptr_t) stress, + page_size, stress_size)) { ok = false; } else if (i == 0) { expected_count = info.count; @@ -584,8 +588,8 @@ int main(void) close(pipefd[0]); int status = 0; bool waited = waitpid(pid, &status, 0) == pid; - EXPECT_TRUE(received && waited && result == 0 && WIFEXITED(status) && - WEXITSTATUS(status) == 0, + EXPECT_TRUE(received && waited && result == 0 && + WIFEXITED(status) && WEXITSTATUS(status) == 0, "fork Shared_Dirty accounting"); } } diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index e926bb1b..91f642d6 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -116,7 +116,7 @@ static void test_growth_preserves_content(void) assert(string_builder_init(&builder, 1) == 0); for (size_t i = 0; i < COUNT; i++) { - expected[i] = (char)('A' + (i % 26)); + expected[i] = (char) ('A' + (i % 26)); char chunk[2] = {expected[i], '\0'}; assert(string_builder_append(&builder, chunk) == 0); } @@ -147,7 +147,8 @@ static void test_overflow_preserves_content(void) char snapshot[32]; assert(string_builder_length(&builder) < sizeof(snapshot)); - memcpy(snapshot, string_builder_data_const(&builder), string_builder_length(&builder)); + memcpy(snapshot, string_builder_data_const(&builder), + string_builder_length(&builder)); size_t old_length = string_builder_length(&builder); size_t old_capacity = string_builder_capacity(&builder); @@ -156,7 +157,8 @@ static void test_overflow_preserves_content(void) assert(errno == EOVERFLOW); assert(string_builder_length(&builder) == old_length); assert(string_builder_capacity(&builder) == old_capacity); - assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); assert(string_builder_data_const(&builder)[old_length] == '\0'); errno = 0; @@ -164,7 +166,8 @@ static void test_overflow_preserves_content(void) assert(errno == EILSEQ); assert(string_builder_length(&builder) == old_length); assert(string_builder_capacity(&builder) == old_capacity); - assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == 0); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); assert(string_builder_data_const(&builder)[old_length] == '\0'); string_builder_destroy(&builder); } diff --git a/tests/test-util.h b/tests/test-util.h index d3aa1c6f..a8b6c146 100644 --- a/tests/test-util.h +++ b/tests/test-util.h @@ -48,7 +48,8 @@ static inline ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) /* Read a file whose size is not available from st_size (for example a proc * file) until EOF, growing the buffer and appending a NUL terminator. */ -static inline ssize_t read_file_dynamic_nul(const char *path, char **buf_out, +static inline ssize_t read_file_dynamic_nul(const char *path, + char **buf_out, size_t *len_out) { if (!path || !buf_out || !len_out) { From e324d2abb06b08f1d0e02983f8df1fa84635494d Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:38:41 +0800 Subject: [PATCH 03/14] Address smaps review comments Apply review feedback to the smaps emulation and its regression tests. --- src/dynamic-array.c | 45 +++++---- src/dynamic-array.h | 19 +++- src/runtime/procemu.c | 96 +++++++++++------- src/string-builder.c | 163 ++++++++++--------------------- src/string-builder.h | 6 +- tests/test-dynamic-array-host.c | 11 +++ tests/test-string-builder-host.c | 28 ++++++ 7 files changed, 201 insertions(+), 167 deletions(-) diff --git a/src/dynamic-array.c b/src/dynamic-array.c index aad9dda4..50db79c9 100644 --- a/src/dynamic-array.c +++ b/src/dynamic-array.c @@ -81,27 +81,31 @@ static int dynamic_array_source_offset(const dynamic_array_t *array, return 1; } -/* Initialize only metadata; allocate storage separately when requested. */ +/* Initialize only metadata; allocate storage separately when requested. + * + * This is a fresh-initialization operation and deliberately does not inspect + * prior object contents, so an automatic, uninitialized object is safe. Use + * dynamic_array_destroy before reinitializing an array that already owns + * storage; otherwise that allocation is intentionally abandoned. + */ int dynamic_array_init(dynamic_array_t *array, size_t element_size) { if (array == NULL || element_size == 0) return dynamic_array_invalid(); - /* There is no allocation to fail here. Preserve storage only when the - * caller is reinitializing an array with the same element type. */ - if (array->data != NULL && array->element_size != element_size) { - errno = EINVAL; - return -1; - } - if (array->data == NULL) { - array->count = 0; - array->capacity = 0; - } - array->element_size = element_size; + *array = (dynamic_array_t) { + .element_size = element_size, + }; return 0; } -/* Initialize and reserve storage for an initial number of elements. */ +/* Initialize and reserve storage for an initial number of elements. + * + * Like dynamic_array_init, this is a fresh-initialization operation that does + * not inspect prior object contents. In particular, do not free an + * indeterminate pointer from an automatic object; destroy an existing array + * before reinitializing it. + */ int dynamic_array_init_with_capacity(dynamic_array_t *array, size_t element_size, size_t initial_capacity) @@ -109,6 +113,11 @@ int dynamic_array_init_with_capacity(dynamic_array_t *array, if (array == NULL || element_size == 0) return dynamic_array_invalid(); + /* Establish a safe zero state before any fallible allocation. This is a + * fresh initializer, so an existing allocation must have been destroyed + * by the caller rather than silently leaked here. */ + *array = (dynamic_array_t) {0}; + size_t bytes; if (initial_capacity != 0 && element_size > SIZE_MAX / initial_capacity) { errno = EOVERFLOW; @@ -124,11 +133,11 @@ int dynamic_array_init_with_capacity(dynamic_array_t *array, } } - free(array->data); - array->data = storage; - array->count = 0; - array->capacity = initial_capacity; - array->element_size = element_size; + *array = (dynamic_array_t) { + .data = storage, + .capacity = initial_capacity, + .element_size = element_size, + }; return 0; } diff --git a/src/dynamic-array.h b/src/dynamic-array.h index 3899194a..fb4c7e3c 100644 --- a/src/dynamic-array.h +++ b/src/dynamic-array.h @@ -43,10 +43,14 @@ static inline int dynamic_array_typed_prepare(dynamic_array_t *array, return was_uninitialized; } -/* Initialize an array for elements of element_size bytes without allocation. */ +/* Initialize an array for elements of element_size bytes without allocation. + * This fresh initializer is safe on an uninitialized automatic object; destroy + * an existing array before reinitializing it. */ int dynamic_array_init(dynamic_array_t *array, size_t element_size); -/* Initialize and reserve initial_capacity element slots. */ +/* Initialize and reserve initial_capacity element slots. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing array before reinitializing it. */ int dynamic_array_init_with_capacity(dynamic_array_t *array, size_t element_size, size_t initial_capacity); @@ -94,19 +98,26 @@ const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); /* Generate a small type-safe facade over the raw container. The facade owns * no additional state; all growth and copying remains in dynamic-array.c. + * A typed object must be zero-initialized before first-touch operations such + * as append, insert, reserve, or resize; the explicit init functions are safe + * on a fresh automatic object and establish the metadata themselves. */ #define DYNAMIC_ARRAY_DEFINE(name, type) \ typedef struct name { \ dynamic_array_t raw; \ } name##_t; \ \ - /* Initialize the typed array with no preallocated storage. */ \ + /* Initialize the typed array with no preallocated storage. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ DYNAMIC_ARRAY_INLINE int name##_init(name##_t *array) \ { \ return dynamic_array_init(array != NULL ? &array->raw : NULL, \ sizeof(type)); \ } \ - /* Initialize typed array and preallocate initial slots. */ \ + /* Initialize typed array and preallocate initial slots. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ DYNAMIC_ARRAY_INLINE int name##_init_with_capacity( \ name##_t *array, size_t initial_capacity) \ { \ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 2359ea56..ab30f02a 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -19,7 +19,8 @@ * bounded. */ #define MAPS_ENTRY_INITIAL_CAP 64 -#define MAPS_ENTRY_MAX (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED) +#define MAPS_ENTRY_MAX \ + (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED * (GUEST_MAX_REGIONS + 1)) /* Bound the transient host-PID snapshot used by /proc/net enumeration. This * is an output-work limit, not the dynamically growing lifecycle-table cap. @@ -170,6 +171,58 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) (void) maps_entries_resize(entries, out + 1); } +/* Add only the portions of a preannounced interval not covered by live VMAs. + * A shadow VMA must never overlap a realized VMA: strict smaps consumers treat + * overlapping headers as a malformed snapshot. Both inputs are page-rounded + * because that is the granularity exposed by /proc/self/maps. */ +static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, + const guest_region_t *shadow, + const guest_region_t *live_regions, + int nlive) +{ + uint64_t shadow_start = shadow->start & ~0xFFFULL; + uint64_t shadow_end = (shadow->end + 0xFFFULL) & ~0xFFFULL; + if (shadow_end <= shadow_start) + return 0; + + uint64_t cursor = shadow_start; + for (int i = 0; i < nlive && cursor < shadow_end; i++) { + uint64_t live_start = live_regions[i].start & ~0xFFFULL; + uint64_t live_end = (live_regions[i].end + 0xFFFULL) & ~0xFFFULL; + if (live_end <= live_start || live_end <= cursor) + continue; + if (live_start >= shadow_end) + break; + + if (live_start > cursor) { + uint64_t gap_end = + live_start < shadow_end ? live_start : shadow_end; + uint64_t offset = shadow->offset; + uint64_t delta = cursor - shadow_start; + if (delta <= UINT64_MAX - offset) + offset += delta; + if (maps_entries_insert_sorted(entries, cursor, gap_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) + return -1; + } + if (live_end > cursor) + cursor = live_end; + } + + if (cursor < shadow_end) { + uint64_t offset = shadow->offset; + uint64_t delta = cursor - shadow_start; + if (delta <= UINT64_MAX - offset) + offset += delta; + if (maps_entries_insert_sorted(entries, cursor, shadow_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) + return -1; + } + return 0; +} + /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily * on first access (Java GC, Go runtime, libnuma probe these to size thread * pools). Layout matches the minimal subset Linux exposes: @@ -2355,9 +2408,8 @@ static int pty_open_master(int linux_flags) } /* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges - * contiguous regions[] runs that came from one mmap, then folds in - * preannounced[] shadow entries whose advertised interval is not yet fully - * covered by live regions. + * contiguous regions[] runs that came from one mmap, then folds in the + * uncovered pieces of preannounced[] shadow entries around live coverage. * * The region and preannounced arrays are mutable from guest mmap/mprotect/ * munmap operations. Snapshot and merge while mmap_lock is held, then release @@ -2404,12 +2456,10 @@ static int proc_build_maps_entries(const guest_t *g, goto out_unlock; } - /* Add preannounced entries only while they still have an uncovered tail. - * Once the union of live regions covers the full advertised interval, - * suppress the shadow entry so /proc/self/maps shows only the realized - * split VMAs. A partial union must stay visible because some - * reserved-but-not-realized span remains to advertise. - */ + /* Add only uncovered portions of each preannounced interval. Keeping the + * shadow VMA whole when a live mapping realizes its middle produces + * overlapping maps/smaps headers; subtract every covered live interval + * instead, preserving any reserved-but-not-realized gaps. */ int npreannounced = g->npreannounced; if (npreannounced < 0) npreannounced = 0; @@ -2417,30 +2467,8 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - bool shadowed = false; - uint64_t covered_end = r->start; - - for (int j = 0; j < nregions; j++) { - const guest_region_t *live = &g->regions[j]; - - if (live->end <= covered_end) - continue; - if (live->start > covered_end) - break; - - covered_end = live->end; - if (covered_end >= r->end) { - shadowed = true; - break; - } - } - - if (shadowed) - continue; - - if (maps_entries_insert_sorted(&entries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, - r->flags, r->offset, r->name) < 0) + if (maps_entries_insert_shadow_gaps(&entries, r, g->regions, nregions) < + 0) goto out_unlock; } maps_entries_merge_adjacent(&entries); diff --git a/src/string-builder.c b/src/string-builder.c index fa134065..c90b8b40 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -20,13 +21,16 @@ static int string_builder_invalid(void) return -1; } -/* Initialize storage and establish an empty, NUL-terminated builder. */ +/* Initialize storage and establish an empty, NUL-terminated builder. This + * fresh initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. + */ int string_builder_init(string_builder_t *builder, size_t initial_capacity) { if (builder == NULL) return string_builder_invalid(); + builder->storage = (string_builder_storage_t) {0}; if (initial_capacity == 0) { - string_builder_destroy(builder); return 0; } if (string_builder_storage_init_with_capacity(&builder->storage, @@ -131,90 +135,19 @@ static int string_builder_format_failure(void) return -1; } -/* Call vsnprintf with a copied argument list for the current sizing pass. */ -static int string_builder_vsnprintf(char *destination, - size_t available, - const char *format, - va_list arguments) -{ - va_list pass; - va_copy(pass, arguments); - errno = 0; -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wformat-nonliteral" - int result = vsnprintf(destination, available, format, pass); -#pragma clang diagnostic pop - va_end(pass); - return result; -} - -typedef enum string_builder_format_status { - STRING_BUILDER_FORMAT_ERROR = -1, - STRING_BUILDER_FORMAT_FIT = 0, - STRING_BUILDER_FORMAT_TRUNCATED = 1 -} string_builder_format_status_t; - -/* Ensure the builder has a writable tail and capture its current length. */ -static int string_builder_prepare_appendf(string_builder_t *builder, - size_t *old_len) -{ - if (string_builder_capacity(builder) == 0 && - string_builder_reserve(builder, 1) < 0) - return -1; - - char *data = string_builder_data(builder); - size_t capacity = string_builder_capacity(builder); - *old_len = string_builder_storage_count(&builder->storage); - if (data == NULL || *old_len >= capacity) - return string_builder_invalid(); - return 0; -} - -/* Format into the current tail and report whether the result was truncated. */ -static string_builder_format_status_t string_builder_try_format( - string_builder_t *builder, - size_t old_len, - const char *format, - va_list arguments, - size_t *written) -{ - char *data = string_builder_data(builder); - size_t available = string_builder_capacity(builder) - old_len; - int formatted = - string_builder_vsnprintf(data + old_len, available, format, arguments); - if (formatted < 0) - return STRING_BUILDER_FORMAT_ERROR; - - size_t visible = strlen(data + old_len); - if ((size_t) formatted < available || visible < available - 1) { - *written = visible; - return STRING_BUILDER_FORMAT_FIT; - } - - *written = (size_t) formatted; - return STRING_BUILDER_FORMAT_TRUNCATED; -} - -/* Restore the terminator after a failed formatting attempt. */ -static void string_builder_rollback_appendf(string_builder_t *builder, - size_t old_len) -{ - char *data = string_builder_data(builder); - if (data != NULL && old_len < string_builder_capacity(builder)) - data[old_len] = '\0'; -} - -/* Append printf-formatted text, growing and retrying when the first pass is - * truncated. +/* Format into separate storage before touching the builder. Besides avoiding + * writes through an aliased format string, this keeps %s arguments that point + * into the builder valid even when appending the result grows the allocation. */ int string_builder_appendf(string_builder_t *builder, const char *format, ...) { - int result = -1; int saved_errno; - size_t old_len = 0; - size_t written = 0; - string_builder_format_status_t status; + int formatted_len; + size_t visible_len; + char *formatted = NULL; va_list arguments; + va_list sizing; + va_list rendering; if (builder == NULL || format == NULL) return string_builder_invalid(); @@ -222,45 +155,57 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) saved_errno = errno; va_start(arguments, format); - if (string_builder_prepare_appendf(builder, &old_len) < 0) + va_copy(sizing, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + formatted_len = vsnprintf(NULL, 0, format, sizing); +#pragma clang diagnostic pop + va_end(sizing); + if (formatted_len < 0) { + string_builder_format_failure(); goto out; + } - status = string_builder_try_format(builder, old_len, format, arguments, - &written); - if (status == STRING_BUILDER_FORMAT_ERROR) { - string_builder_format_failure(); - goto rollback; + formatted = malloc((size_t) formatted_len + 1); + if (formatted == NULL) { + errno = ENOMEM; + goto out; } - if (status == STRING_BUILDER_FORMAT_TRUNCATED) { - if (string_builder_reserve(builder, written) < 0) - goto rollback; - status = string_builder_try_format(builder, old_len, format, arguments, - &written); - if (status == STRING_BUILDER_FORMAT_ERROR) { - string_builder_format_failure(); - goto rollback; - } - if (status == STRING_BUILDER_FORMAT_TRUNCATED) { - errno = EOVERFLOW; - goto rollback; - } + va_copy(rendering, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + int rendered_len = + vsnprintf(formatted, (size_t) formatted_len + 1, format, rendering); +#pragma clang diagnostic pop + va_end(rendering); + if (rendered_len < 0 || rendered_len != formatted_len) { + string_builder_format_failure(); + goto out; } - if (string_builder_commit_append( - builder, string_builder_data(builder) + old_len, written) < 0) - goto rollback; + /* Preserve the builder's C-string semantics for formatted NUL bytes. */ + visible_len = strlen(formatted); + if (visible_len == 0) { + errno = saved_errno; + free(formatted); + va_end(arguments); + return 0; + } + if (string_builder_commit_append(builder, formatted, visible_len) < 0) + goto out; errno = saved_errno; - result = 0; - goto out; - -rollback: - string_builder_rollback_appendf(builder, old_len); + free(formatted); + va_end(arguments); + return 0; out: + free(formatted); va_end(arguments); - return result; + return -1; } /* Return mutable storage for the builder, if it has been allocated. */ diff --git a/src/string-builder.h b/src/string-builder.h index f9fb1c0c..c0942071 100644 --- a/src/string-builder.h +++ b/src/string-builder.h @@ -22,8 +22,10 @@ typedef struct string_builder { } string_builder_t; /* Initialize a builder, reserving initial_capacity bytes including the NUL. - * A zero capacity leaves storage unallocated for lazy growth. Returns 0 on - * success or -1 with errno set on invalid input or allocation failure. + * A zero capacity leaves storage unallocated for lazy growth. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. Returns 0 on success or -1 with + * errno set on invalid input or allocation failure. */ int string_builder_init(string_builder_t *builder, size_t initial_capacity); diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c index 6a3ea3ce..0ca09eb1 100644 --- a/tests/test-dynamic-array-host.c +++ b/tests/test-dynamic-array-host.c @@ -23,6 +23,17 @@ DYNAMIC_ARRAY_DEFINE(odd_array, odd_t) static void test_zero_and_init(void) { + int_array_t fresh; + assert(int_array_init_with_capacity(&fresh, 4) == 0); + assert(int_array_capacity(&fresh) >= 4); + int_array_destroy(&fresh); + + int_array_t fresh_lazy; + assert(int_array_init(&fresh_lazy) == 0); + assert(int_array_append_value(&fresh_lazy, 11) == 0); + assert(*int_array_at(&fresh_lazy, 0) == 11); + int_array_destroy(&fresh_lazy); + int_array_t values = {0}; int value = 7; assert(int_array_count(&values) == 0); diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index 91f642d6..073958ed 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -29,10 +29,24 @@ static void expect_string(const string_builder_t *builder, const char *expected) static void test_zero_and_initial_capacity(void) { + /* Fresh automatic objects need not be manually zeroed before init. */ + string_builder_t fresh; + assert(string_builder_init(&fresh, 0) == 0); + assert(string_builder_append(&fresh, "fresh") == 0); + expect_string(&fresh, "fresh"); + string_builder_destroy(&fresh); + + string_builder_t fresh_capacity; + assert(string_builder_init(&fresh_capacity, 32) == 0); + assert(string_builder_capacity(&fresh_capacity) >= 32); + string_builder_destroy(&fresh_capacity); + string_builder_t zero = {0}; assert(string_builder_data(&zero) == NULL); assert(string_builder_length(&zero) == 0); assert(string_builder_capacity(&zero) == 0); + assert(string_builder_appendf(&zero, "%c", '\0') == 0); + assert(string_builder_data(&zero) == NULL); /* The public API also accepts a plain {0} value without an init call. */ assert(string_builder_append(&zero, "zero") == 0); expect_string(&zero, "zero"); @@ -139,6 +153,19 @@ static void test_alias_append(void) string_builder_destroy(&builder); } +static void test_formatted_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "x%s") == 0); + const char *alias = string_builder_data_const(&builder); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + assert(string_builder_appendf(&builder, alias, alias) == 0); +#pragma clang diagnostic pop + expect_string(&builder, "x%sxx%s"); + string_builder_destroy(&builder); +} + static void test_overflow_preserves_content(void) { string_builder_t builder = {0}; @@ -179,6 +206,7 @@ int main(void) test_c_string_semantics(); test_growth_preserves_content(); test_alias_append(); + test_formatted_alias_append(); test_overflow_preserves_content(); puts("test-string-builder-host: PASS"); return 0; From e82e45621631f42841cda1875c0df25ad56ec552 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:45:08 +0800 Subject: [PATCH 04/14] Fix string builder capacity warning Grow formatted string storage safely and silence the Infer warning exposed by the smaps changes. --- src/string-builder.c | 8 +++++++- tests/test-string-builder-host.c | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/string-builder.c b/src/string-builder.c index c90b8b40..97d7bec9 100644 --- a/src/string-builder.c +++ b/src/string-builder.c @@ -148,6 +148,7 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) va_list arguments; va_list sizing; va_list rendering; + char sizing_sink; if (builder == NULL || format == NULL) return string_builder_invalid(); @@ -159,7 +160,10 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) errno = 0; #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wformat-nonliteral" - formatted_len = vsnprintf(NULL, 0, format, sizing); + /* A non-NULL destination keeps static analyzers from treating the + * standards-sanctioned n == 0 sizing call as a null dereference. The + * destination is never written when its size is zero. */ + formatted_len = vsnprintf(&sizing_sink, 0, format, sizing); #pragma clang diagnostic pop va_end(sizing); if (formatted_len < 0) { @@ -194,6 +198,8 @@ int string_builder_appendf(string_builder_t *builder, const char *format, ...) va_end(arguments); return 0; } + if (string_builder_reserve(builder, visible_len) < 0) + goto out; if (string_builder_commit_append(builder, formatted, visible_len) < 0) goto out; diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c index 073958ed..e404680c 100644 --- a/tests/test-string-builder-host.c +++ b/tests/test-string-builder-host.c @@ -122,6 +122,22 @@ static void test_c_string_semantics(void) string_builder_destroy(&fit); } +static void test_formatted_append_reserves_terminator(void) +{ + string_builder_t builder = {0}; + + /* A formatted append must reserve one byte beyond the visible payload for + * the builder's trailing NUL. This exact-length payload used to make the + * generic array allocate only the payload bytes before the terminator was + * written. + */ + assert(string_builder_appendf(&builder, "%s", "12345678") == 0); + expect_string(&builder, "12345678"); + assert(string_builder_capacity(&builder) >= + string_builder_length(&builder) + 1); + string_builder_destroy(&builder); +} + static void test_growth_preserves_content(void) { enum { COUNT = 4096 }; @@ -204,6 +220,7 @@ int main(void) test_zero_and_initial_capacity(); test_text_and_formatted_append(); test_c_string_semantics(); + test_formatted_append_reserves_terminator(); test_growth_preserves_content(); test_alias_append(); test_formatted_alias_append(); From 64560beffa89ca7491317034977d463c1411299b Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:50:26 +0800 Subject: [PATCH 05/14] Use unique names for concurrent shm tests Avoid collisions when concurrent shared-memory tests create temporary names. --- tests/test-dev-shm-paths.c | 53 ++++++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/tests/test-dev-shm-paths.c b/tests/test-dev-shm-paths.c index 728aae9d..bcb68252 100644 --- a/tests/test-dev-shm-paths.c +++ b/tests/test-dev-shm-paths.c @@ -51,19 +51,42 @@ static char shm_dir[128]; static char shm_fifo[128]; static char shm_exec[128]; static char victim_path[128]; +static char fixture_suffix[16]; -static void name_fixtures(void) +/* Guest PIDs restart from the same value in each independent elfuse process. + * Reserve a host-unique suffix so concurrent runtime jobs cannot unlink or + * replace one another's /dev/shm fixtures. */ +static int name_fixtures(void) { - int pid = (int) getpid(); - snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%d", pid); - snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%d", pid); - snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%d", pid); - snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%d", pid); - snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%d", pid); - snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%d", pid); - snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%d", pid); - snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%d", - pid); + char seed[] = "/tmp/elfuse-shm-seed-XXXXXX"; + int fd = mkstemp(seed); + if (fd < 0) + return -1; + close(fd); + (void) unlink(seed); + + const char *suffix = strrchr(seed, '-'); + if (suffix == NULL || suffix[1] == '\0') + return -1; + snprintf(fixture_suffix, sizeof(fixture_suffix), "%s", suffix + 1); + + snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%s", + fixture_suffix); + snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%s", + fixture_suffix); + snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%s", + fixture_suffix); + snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%s", + fixture_suffix); + snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", + fixture_suffix); + snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%s", + fixture_suffix); + snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%s", + fixture_suffix); + snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%s", + fixture_suffix); + return 0; } static void cleanup_fixtures(void) @@ -724,7 +747,7 @@ static void test_dotdot_bearing_names_allowed(void) static const char *names[] = {"a..b", "..a", "a..", "..."}; for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { char p[128]; - snprintf(p, sizeof(p), SHM_DIR "elfuse_%d_%s", (int) getpid(), + snprintf(p, sizeof(p), SHM_DIR "elfuse_%s_%s", fixture_suffix, names[i]); int fd = open(p, O_CREAT | O_EXCL | O_RDWR, 0600); if (fd < 0) { @@ -775,7 +798,11 @@ int main(int argc, char **argv) printf("test-dev-shm-paths: /dev/shm path-syscall consistency\n"); - name_fixtures(); + if (name_fixtures() < 0) { + FAIL("reserve unique fixture suffix"); + SUMMARY("test-dev-shm-paths"); + return 1; + } cleanup_fixtures(); if (test_open_then_chmod() == 0) { From 40a79f4cfadd7201967128cf1735659e8ef69734 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 09:52:35 +0800 Subject: [PATCH 06/14] Format concurrent shm fixture setup Keep concurrent shared-memory fixture setup consistent with repository formatting conventions. --- tests/test-dev-shm-paths.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test-dev-shm-paths.c b/tests/test-dev-shm-paths.c index bcb68252..dff9e9ff 100644 --- a/tests/test-dev-shm-paths.c +++ b/tests/test-dev-shm-paths.c @@ -78,8 +78,7 @@ static int name_fixtures(void) fixture_suffix); snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%s", fixture_suffix); - snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", - fixture_suffix); + snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", fixture_suffix); snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%s", fixture_suffix); snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%s", From f53ed11cc7f51c959d8e111e6efe80068be7b103 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 11:34:18 +0800 Subject: [PATCH 07/14] Optimize fragmented proc snapshots Append live and shadow VMAs, then sort and merge once. This keeps fragmented maps and smaps snapshots from quadratic insertion work. --- src/runtime/procemu.c | 73 +++++++++++++++++++++++++++++-------------- 1 file changed, 50 insertions(+), 23 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index ab30f02a..2414ac76 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -103,19 +103,24 @@ typedef struct { int prot, flags; uint64_t offset; char name[64]; + /* Preserve the producer order for equal-start entries when qsort() is + * used below. Existing snapshots placed equal-start entries after one + * another in append order, and keeping that order avoids changing the + * handling of malformed/overlapping shadow metadata. */ + size_t order; } maps_entry_t; /* A growable VMA snapshot. The generated facade keeps element-size and * allocation bookkeeping in the generic array implementation. */ DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) -static int maps_entries_insert_sorted(maps_entries_t *entries, - uint64_t start, - uint64_t end, - int prot, - int flags, - uint64_t offset, - const char *name) +static int maps_entries_append_entry(maps_entries_t *entries, + uint64_t start, + uint64_t end, + int prot, + int flags, + uint64_t offset, + const char *name) { if (end <= start) return 0; @@ -133,17 +138,34 @@ static int maps_entries_insert_sorted(maps_entries_t *entries, .prot = prot, .flags = flags, .offset = offset, + .order = maps_entries_count(entries), }; if (name && name[0]) str_copy_trunc(value.name, name, sizeof(value.name)); else value.name[0] = '\0'; - size_t i = maps_entries_count(entries); - while (i > 0 && maps_entries_at(entries, i - 1)->start > start) { - i--; - } - return maps_entries_insert(entries, i, &value); + return maps_entries_append_value(entries, value); +} + +/* The live-region and shadow-gap producers are each ordered, but their + * outputs interleave. Append both streams while holding mmap_lock and sort + * once after all gaps have been generated. This avoids shifting an already + * populated array for every split shadow gap (the old insertion path was + * quadratic for fragmented snapshots). */ +static int maps_entries_compare_start(const void *lhs, const void *rhs) +{ + const maps_entry_t *a = lhs; + const maps_entry_t *b = rhs; + if (a->start < b->start) + return -1; + if (a->start > b->start) + return 1; + if (a->order < b->order) + return -1; + if (a->order > b->order) + return 1; + return 0; } static void maps_entries_merge_adjacent(maps_entries_t *entries) @@ -175,7 +197,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) * A shadow VMA must never overlap a realized VMA: strict smaps consumers treat * overlapping headers as a malformed snapshot. Both inputs are page-rounded * because that is the granularity exposed by /proc/self/maps. */ -static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, +static int maps_entries_append_shadow_gaps(maps_entries_t *entries, const guest_region_t *shadow, const guest_region_t *live_regions, int nlive) @@ -201,9 +223,9 @@ static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_insert_sorted(entries, cursor, gap_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, gap_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) return -1; } if (live_end > cursor) @@ -215,9 +237,9 @@ static int maps_entries_insert_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_insert_sorted(entries, cursor, shadow_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, shadow_end, + shadow->prot, shadow->flags, offset, + shadow->name) < 0) return -1; } return 0; @@ -2410,6 +2432,8 @@ static int pty_open_master(int linux_flags) /* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges * contiguous regions[] runs that came from one mmap, then folds in the * uncovered pieces of preannounced[] shadow entries around live coverage. + * Producers append entries while the lock is held; one sort/merge pass puts + * the interleaved live and shadow streams back into VMA order. * * The region and preannounced arrays are mutable from guest mmap/mprotect/ * munmap operations. Snapshot and merge while mmap_lock is held, then release @@ -2451,8 +2475,8 @@ static int proc_build_maps_entries(const guest_t *g, last->end = end; continue; } - if (maps_entries_insert_sorted(&entries, start, end, r->prot, r->flags, - r->offset, r->name) < 0) + if (maps_entries_append_entry(&entries, start, end, r->prot, r->flags, + r->offset, r->name) < 0) goto out_unlock; } @@ -2467,10 +2491,13 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - if (maps_entries_insert_shadow_gaps(&entries, r, g->regions, nregions) < - 0) + if (maps_entries_append_shadow_gaps(&entries, r, g->regions, + nregions) < 0) goto out_unlock; } + if (maps_entries_count(&entries) > 1) + qsort(maps_entries_data(&entries), maps_entries_count(&entries), + sizeof(maps_entry_t), maps_entries_compare_start); maps_entries_merge_adjacent(&entries); result = (int) maps_entries_count(&entries); From c71476ffb3ae683714e03474a7e58b489206ab85 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 11:43:13 +0800 Subject: [PATCH 08/14] Format proc VMA snapshot calls Apply repository formatting conventions to proc VMA snapshot calls. --- src/runtime/procemu.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 2414ac76..972a75b3 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -237,9 +237,8 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, uint64_t delta = cursor - shadow_start; if (delta <= UINT64_MAX - offset) offset += delta; - if (maps_entries_append_entry(entries, cursor, shadow_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + if (maps_entries_append_entry(entries, cursor, shadow_end, shadow->prot, + shadow->flags, offset, shadow->name) < 0) return -1; } return 0; @@ -2491,8 +2490,8 @@ static int proc_build_maps_entries(const guest_t *g, npreannounced = GUEST_MAX_PREANNOUNCED; for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - if (maps_entries_append_shadow_gaps(&entries, r, g->regions, - nregions) < 0) + if (maps_entries_append_shadow_gaps(&entries, r, g->regions, nregions) < + 0) goto out_unlock; } if (maps_entries_count(&entries) > 1) From 5f48a8d2c8fcd928179dd7147d39edf74b6c2d54 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 15:47:31 +0800 Subject: [PATCH 09/14] Make smaps Shared_Dirty fork-aware Track whether each guest VMA existed at the fork snapshot so post-fork mappings are excluded from the synthetic Shared_Dirty compatibility signal. Propagate the metadata through fork IPC and memory-region transformations, and add a regression test for post-fork mappings. --- docs/internals.md | 15 +-- src/core/guest.c | 16 +++- src/core/guest.h | 15 ++- src/runtime/fork-state.c | 7 ++ src/runtime/fork-state.h | 2 +- src/runtime/procemu.c | 86 ++++++++++++----- src/syscall/mem.c | 138 ++++++++++++++++++++++------ tests/test-fork-ipc-protocol-host.c | 7 +- tests/test-matrix.sh | 4 +- tests/test-proc-smap.c | 22 ++++- 10 files changed, 237 insertions(+), 75 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index 1aa36950..3096dc26 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -808,18 +808,21 @@ ProtectionKey, VmFlags `Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a -non-zero value: in a fork child, writable private anonymous VMAs report their -full VMA size because they are logically shared with the parent's CoW -snapshot. Every other numeric counter is emitted as zero, including -`THPeligible` and `ProtectionKey`. `VmFlags` is evidence-based and contains -only the permission/sharing flags represented by the tracked VMA (`rd`, `wr`, +non-zero value: in a fork child, writable private anonymous VMAs that existed +in the parent's CoW snapshot report their full VMA size because they are +logically shared with that snapshot. Every other numeric counter, including +`THPeligible` and `ProtectionKey`, is emitted as zero. `VmFlags` is +evidence-based and contains only the permission/sharing flags represented by +the tracked VMA (`rd`, `wr`, `ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing against a real Linux kernel should tolerate either conditional field being absent. These are coarse VMA-level values, not host page-residency, dirty-bit, or proportional-sharing accounting, so they are suitable for fork-safety checks -but not precise memory profiling. `smaps_rollup` is not implemented. +but not precise memory profiling. The fork-child marker is tracked per VMA, so +writable private anonymous mappings created after fork are excluded from the +compatibility signal. `smaps_rollup` is not implemented. Synthetic proc directories have explicit snapshot boundaries. The backing trees reached by opening `/proc` or `/proc/self` are materialized once, on the diff --git a/src/core/guest.c b/src/core/guest.c index addc79f4..575d5d8d 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1870,6 +1870,8 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return false; if (a->backing_ro != b->backing_ro) return false; + if (a->inherited_at_fork != b->inherited_at_fork) + return false; if (a->overlay_active || b->overlay_active) return false; if (strcmp(a->name, b->name) != 0) @@ -1981,7 +1983,7 @@ int guest_region_add_ex(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false); } int guest_region_add_ex_gpa(guest_t *g, @@ -2002,7 +2004,7 @@ int guest_region_add_ex_gpa(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, gpa_base, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false); } int guest_region_add_ex_owned(guest_t *g, @@ -2012,10 +2014,12 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork) { return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, + inherited_at_fork); } int guest_region_add_ex_owned_gpa(guest_t *g, @@ -2026,7 +2030,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2055,6 +2060,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->shared = (flags & LINUX_MAP_SHARED) != 0; r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; + r->inherited_at_fork = inherited_at_fork; guest_region_clear_overlay(r); if (name) { str_copy_trunc(r->name, name, sizeof(r->name)); diff --git a/src/core/guest.h b/src/core/guest.h index 5e80d9ee..65267cda 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -242,6 +242,11 @@ typedef struct { * later PROT_WRITE request against it with EACCES, * matching a real kernel's VMA max_prot tracking. */ + bool inherited_at_fork; /* Region existed at the most recent fork + * snapshot. Used by synthetic smaps to report + * only VMAs that can participate in that fork's + * CoW snapshot as Shared_Dirty. + */ bool overlay_active; /* Region has a live host MAP_FIXED|MAP_SHARED overlay * of backing_fd at host_base+start. The kernel's page * cache keeps it coherent with the file and with peer @@ -1176,7 +1181,9 @@ int guest_region_add_ex_gpa(guest_t *g, int backing_fd); /* Like guest_region_add_ex, but consumes owned_backing_fd on success or - * failure. + * failure. inherited_at_fork identifies bytes copied from the latest fork + * snapshot; ordinary mmap/exec callers pass false, while fork/mremap restore + * paths preserve the source VMA's value. */ int guest_region_add_ex_owned(guest_t *g, uint64_t start, @@ -1185,7 +1192,8 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork); int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t start, uint64_t end, @@ -1194,7 +1202,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork); /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 43b75d49..9ff5a799 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -926,6 +926,13 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) return -1; g->nregions = (int) recv_regions; + /* Every VMA present in the serialized parent snapshot is inherited by + * this child, regardless of whether the parent itself created it after an + * earlier fork. New mappings added in this process start unmarked through + * guest_region_add_ex_owned[_gpa]. + */ + for (int i = 0; i < g->nregions; i++) + g->regions[i].inherited_at_fork = true; g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index d4a5663e..b6f8331e 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464EU /* "ELFN" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464FU /* "ELFO" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 972a75b3..97911b31 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -102,6 +102,7 @@ typedef struct { uint64_t start, end; int prot, flags; uint64_t offset; + bool inherited_at_fork; char name[64]; /* Preserve the producer order for equal-start entries when qsort() is * used below. Existing snapshots placed equal-start entries after one @@ -114,13 +115,45 @@ typedef struct { * allocation bookkeeping in the generic array implementation. */ DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) +/* Round a VMA endpoint up to the 4 KiB granularity exposed by procfs without + * allowing the addition to wrap. There is no representable page-aligned + * endpoint above UINT64_MAX - 0xFFF, so saturate to UINT64_MAX and let the + * caller's normal end <= start validation handle an empty interval. + */ +static uint64_t maps_align_up_page(uint64_t value) +{ + const uint64_t mask = 0xFFFULL; + if (value > UINT64_MAX - mask) + return UINT64_MAX; + return (value + mask) & ~mask; +} + +/* Translate a shadow VMA's cursor into its file offset. A wrapped offset would + * produce a syntactically valid but semantically wrong maps entry, so surface + * the overflow to the intercepted open instead. + */ +static int maps_shadow_offset(const guest_region_t *shadow, + uint64_t shadow_start, + uint64_t cursor, + uint64_t *offset_out) +{ + uint64_t delta = cursor - shadow_start; + if (delta > UINT64_MAX - shadow->offset) { + errno = EOVERFLOW; + return -1; + } + *offset_out = shadow->offset + delta; + return 0; +} + static int maps_entries_append_entry(maps_entries_t *entries, uint64_t start, uint64_t end, int prot, int flags, uint64_t offset, - const char *name) + const char *name, + bool inherited_at_fork) { if (end <= start) return 0; @@ -138,6 +171,7 @@ static int maps_entries_append_entry(maps_entries_t *entries, .prot = prot, .flags = flags, .offset = offset, + .inherited_at_fork = inherited_at_fork, .order = maps_entries_count(entries), }; if (name && name[0]) @@ -182,6 +216,7 @@ static void maps_entries_merge_adjacent(maps_entries_t *entries) current->prot == previous->prot && current->flags == previous->flags && current->offset == previous->offset && + current->inherited_at_fork == previous->inherited_at_fork && strcmp(current->name, previous->name) == 0) { previous->end = current->end; continue; @@ -203,14 +238,14 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, int nlive) { uint64_t shadow_start = shadow->start & ~0xFFFULL; - uint64_t shadow_end = (shadow->end + 0xFFFULL) & ~0xFFFULL; + uint64_t shadow_end = maps_align_up_page(shadow->end); if (shadow_end <= shadow_start) return 0; uint64_t cursor = shadow_start; for (int i = 0; i < nlive && cursor < shadow_end; i++) { uint64_t live_start = live_regions[i].start & ~0xFFFULL; - uint64_t live_end = (live_regions[i].end + 0xFFFULL) & ~0xFFFULL; + uint64_t live_end = maps_align_up_page(live_regions[i].end); if (live_end <= live_start || live_end <= cursor) continue; if (live_start >= shadow_end) @@ -219,13 +254,12 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, if (live_start > cursor) { uint64_t gap_end = live_start < shadow_end ? live_start : shadow_end; - uint64_t offset = shadow->offset; - uint64_t delta = cursor - shadow_start; - if (delta <= UINT64_MAX - offset) - offset += delta; - if (maps_entries_append_entry(entries, cursor, gap_end, - shadow->prot, shadow->flags, offset, - shadow->name) < 0) + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; + if (maps_entries_append_entry( + entries, cursor, gap_end, shadow->prot, shadow->flags, + offset, shadow->name, shadow->inherited_at_fork) < 0) return -1; } if (live_end > cursor) @@ -233,12 +267,12 @@ static int maps_entries_append_shadow_gaps(maps_entries_t *entries, } if (cursor < shadow_end) { - uint64_t offset = shadow->offset; - uint64_t delta = cursor - shadow_start; - if (delta <= UINT64_MAX - offset) - offset += delta; + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; if (maps_entries_append_entry(entries, cursor, shadow_end, shadow->prot, - shadow->flags, offset, shadow->name) < 0) + shadow->flags, offset, shadow->name, + shadow->inherited_at_fork) < 0) return -1; } return 0; @@ -2449,6 +2483,7 @@ static int proc_build_maps_entries(const guest_t *g, maps_entries_t entries = {0}; int result = -1; + int saved_errno = 0; pthread_mutex_lock(&mmap_lock); @@ -2464,18 +2499,20 @@ static int proc_build_maps_entries(const guest_t *g, for (int i = 0; i < nregions; i++) { const guest_region_t *r = &g->regions[i]; uint64_t start = r->start & ~0xFFFULL; - uint64_t end = (r->end + 0xFFFULL) & ~0xFFFULL; + uint64_t end = maps_align_up_page(r->end); size_t count = maps_entries_count(&entries); maps_entry_t *last = count > 0 ? maps_entries_at(&entries, count - 1) : NULL; if (last != NULL && last->end == start && last->prot == r->prot && last->flags == r->flags && last->offset == r->offset && + last->inherited_at_fork == r->inherited_at_fork && !strcmp(last->name, r->name)) { last->end = end; continue; } if (maps_entries_append_entry(&entries, start, end, r->prot, r->flags, - r->offset, r->name) < 0) + r->offset, r->name, + r->inherited_at_fork) < 0) goto out_unlock; } @@ -2501,10 +2538,11 @@ static int proc_build_maps_entries(const guest_t *g, result = (int) maps_entries_count(&entries); out_unlock: + saved_errno = errno; pthread_mutex_unlock(&mmap_lock); if (result < 0) { maps_entries_destroy(&entries); - errno = ENOMEM; + errno = saved_errno; return -1; } *entries_out = entries; @@ -2609,9 +2647,10 @@ static int proc_open_self_maps(const guest_t *g) /* Emit a Linux-shaped /proc/self/smaps approximation. The runtime tracks * guest VMAs and logical fork snapshots, but it cannot observe kernel page - * residency or dirty bits on the host. For a fork child, writable private - * anonymous VMAs therefore report their full VMA size as Shared_Dirty; all - * other fields that require kernel page accounting are stable zeroes. + * residency or dirty bits on the host. Writable private anonymous VMAs that + * were present in the most recent fork snapshot report their full VMA size as + * Shared_Dirty; newly-created VMAs are excluded. All other fields that + * require kernel page accounting are stable zeroes. */ static int proc_open_self_smaps(const guest_t *g) { @@ -2626,7 +2665,6 @@ static int proc_open_self_smaps(const guest_t *g) if (string_builder_init(&builder, initial_capacity) < 0) goto out; - bool fork_child = proc_get_ppid() != 0; for (int i = 0; i < nentries; i++) { const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char header[256]; @@ -2640,8 +2678,8 @@ static int proc_open_self_smaps(const guest_t *g) bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; bool private_anon = (e->flags & LINUX_MAP_PRIVATE) && anonymous && !shared; - bool logical_shared_dirty = - fork_child && private_anon && (e->prot & LINUX_PROT_WRITE); + bool logical_shared_dirty = e->inherited_at_fork && private_anon && + (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; char vmflags[64]; diff --git a/src/syscall/mem.c b/src/syscall/mem.c index a5477c5e..8fc47d90 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -83,6 +83,7 @@ typedef struct { uint64_t overlay_start; uint64_t overlay_end; bool backing_ro; + bool inherited_at_fork; char name[sizeof(((guest_region_t *) 0)->name)]; } region_snapshot_t; @@ -207,19 +208,67 @@ static void mark_overlay_metadata_range(guest_t *g, } } -/* Mark the region spanning exactly [start, end) as backed by a fd that lost - * write access, so sys_mprotect rejects a later PROT_WRITE upgrade. Exact match - * (not overlap) because callers use this right after installing a single - * freshly-added region. +/* Mark every region overlapping [start, end) as backed by a fd that lost write + * access, so sys_mprotect rejects a later PROT_WRITE upgrade. mremap can split + * an inherited VMA at the fork boundary, so callers must not require one + * exact region match here. */ static void mark_region_backing_ro(guest_t *g, uint64_t start, uint64_t end) { for (int i = 0; i < g->nregions; i++) { - if (g->regions[i].start == start && g->regions[i].end == end) { - g->regions[i].backing_ro = true; + if (g->regions[i].start >= end) break; + if (g->regions[i].end <= start) + continue; + g->regions[i].backing_ro = true; + } +} + +/* Track an mremap result without losing the fork boundary inside an in-place + * growth or a moved mapping. Bytes copied from the old VMA were present at the + * fork snapshot; an extension is new child-private address space and must + * remain unmarked. Keep the two portions as separate VMAs even when all other + * metadata matches so the synthetic smaps view can distinguish them. + */ +static int add_mremap_region(guest_t *g, + uint64_t start, + uint64_t old_size, + uint64_t new_size, + int prot, + int flags, + uint64_t offset, + const char *name, + int backing_fd, + bool inherited_at_fork) +{ + if (inherited_at_fork && old_size > 0 && old_size < new_size) { + int tail_backing_fd = -1; + if (backing_fd >= 0) { + tail_backing_fd = dup(backing_fd); + if (tail_backing_fd < 0) { + close(backing_fd); + return -1; + } + } + if (guest_region_add_ex_owned(g, start, start + old_size, prot, flags, + offset, name, backing_fd, true) < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return -1; + } + + if (guest_region_add_ex_owned(g, start + old_size, start + new_size, + prot, flags, offset + old_size, name, + tail_backing_fd, false) < 0) { + guest_region_remove(g, start, start + old_size); + return -1; } + return 0; } + + return guest_region_add_ex_owned(g, start, start + new_size, prot, flags, + offset, name, backing_fd, + inherited_at_fork && old_size > 0); } static void region_clip_overlay(guest_region_t *r) @@ -851,8 +900,8 @@ static int64_t sys_mmap_high_va(guest_t *g, replaced_region_removed = true; } if (guest_region_add_ex_owned_gpa(g, addr, addr + length, gpa_base, prot, - flags, offset, NULL, - track_backing_fd) < 0) + flags, offset, NULL, track_backing_fd, + false) < 0) goto fail; /* Ownership of track_backing_fd is now held by the new region. The fail * handler below skips closing when track_backing_fd < 0, so subsequent @@ -1133,6 +1182,7 @@ static int capture_region_snapshots(guest_t *g, snap->overlay_start = r->overlay_start; snap->overlay_end = r->overlay_end; snap->backing_ro = r->backing_ro; + snap->inherited_at_fork = r->inherited_at_fork; str_copy_trunc(snap->name, r->name, sizeof(snap->name)); } @@ -1239,7 +1289,7 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) if (guest_region_add_ex_owned_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd) < 0) { + snap->backing_fd, snap->inherited_at_fork) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -1829,7 +1879,26 @@ int64_t sys_brk(guest_t *g, uint64_t addr) for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start == g->brk_base && !strcmp(g->regions[i].name, "[heap]")) { - g->regions[i].end = new_off; + guest_region_t *heap = &g->regions[i]; + uint64_t old_heap_end = heap->end; + if (new_off > old_heap_end && heap->inherited_at_fork) { + /* Keep the fork-snapshot portion separate from pages + * materialized by this post-fork brk growth. If the + * semantic table is full, retain the historical + * conservative range and mark it stale rather than + * failing an otherwise successful brk syscall. + */ + if (guest_region_add( + g, old_heap_end, new_off, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, + "[heap]") < 0) { + heap->end = new_off; + g->regions_tracker_stale = true; + } + } else { + heap->end = new_off; + } found = true; break; } @@ -2559,7 +2628,7 @@ int64_t sys_mmap(guest_t *g, */ if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, track_backing_fd) < 0) { + NULL, track_backing_fd, false) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -2760,8 +2829,6 @@ int64_t sys_mremap(guest_t *g, {old_off, old_end}, {new_off, new_end}, }; - if (!region_has_capacity_after_removes(g, removed, 2, 1)) - return -LINUX_ENOMEM; /* Capture old region metadata BEFORE modifying any regions. If mremap * removed destination first, an overlapping source would lose its @@ -2779,6 +2846,14 @@ int64_t sys_mremap(guest_t *g, return -LINUX_ENOMEM; bool source_overlay = old_reg && region_has_live_overlay(old_reg); bool source_backing_ro = old_reg && old_reg->backing_ro; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; + int added_regions = + source_inherited_at_fork && old_size < new_size ? 2 : 1; + if (!region_has_capacity_after_removes(g, removed, 2, added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + return -LINUX_ENOMEM; + } uint64_t source_file_off = old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; char track_name[sizeof(old_reg->name)] = {0}; @@ -2948,9 +3023,9 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; } - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) { + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork) < 0) { (void) restore_region_snapshots(g, dest_snaps, dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); @@ -2993,9 +3068,6 @@ int64_t sys_mremap(guest_t *g, if (can_grow) { remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) - return -LINUX_ENOMEM; - /* Extend in place */ const guest_region_t *old_reg = guest_region_find(g, old_off); int prot = old_reg ? old_reg->prot @@ -3011,6 +3083,15 @@ int64_t sys_mremap(guest_t *g, uint64_t old_overlay_end = old_overlay ? old_reg->overlay_end : 0; bool old_backing_ro = old_reg && old_reg->backing_ro; + bool old_inherited_at_fork = + old_reg && old_reg->inherited_at_fork; + int added_regions = old_inherited_at_fork ? 2 : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, + added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + return -LINUX_ENOMEM; + } if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) return -LINUX_ENOMEM; char track_name[sizeof(old_reg->name)] = {0}; @@ -3028,10 +3109,10 @@ int64_t sys_mremap(guest_t *g, /* Update region tracking: remove old, add extended */ guest_region_remove(g, old_off, old_off + old_size); - if (guest_region_add_ex_owned(g, old_off, old_off + new_size, - prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, - track_backing_fd) < 0) + if (add_mremap_region( + g, old_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, old_inherited_at_fork) < 0) return -LINUX_ENOMEM; if (old_overlay) mark_overlay_metadata_range(g, old_off, old_off + old_size, @@ -3073,6 +3154,7 @@ int64_t sys_mremap(guest_t *g, source_overlay ? old_reg->overlay_start : 0; uint64_t source_overlay_end = source_overlay ? old_reg->overlay_end : 0; bool source_backing_ro = old_reg && old_reg->backing_ro; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; uint64_t source_file_off = old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; uint64_t source_overlay_file_off = @@ -3104,7 +3186,9 @@ int64_t sys_mremap(guest_t *g, } remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) { + int added_regions = + source_inherited_at_fork && old_size < new_size ? 2 : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); return -LINUX_ENOMEM; @@ -3189,9 +3273,9 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; /* Track new region */ - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork) < 0) return -LINUX_ENOMEM; if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 8dc4cc88..0f420098 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -19,9 +19,10 @@ #define LEGACY_ELFK_MAGIC 0x454C464BU #define PREVIOUS_ELFL_MAGIC 0x454C464CU #define PREVIOUS_ELFM_MAGIC 0x454C464DU +#define PREVIOUS_ELFN_MAGIC 0x454C464EU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464EU, - "fork IPC protocol magic must remain ELFN until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464FU, + "fork IPC protocol magic must remain ELFO until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -31,6 +32,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFL_MAGIC, "NOFILE header fields require rejecting ELFL peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFM_MAGIC, "start_stack header field requires rejecting ELFM peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFN_MAGIC, + "region fork metadata requires rejecting ELFN peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 3c9eaaf4..fcddc3ee 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -353,8 +353,8 @@ QEMU_SKIP=" # under elfuse, while a real kernel allows the open and only rejects the # write -- a genuine behavioral difference worth reviewing on its own, # not just an environment artifact. -# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including a -# coarse Shared_Dirty compatibility signal for untouched writable VMAs. Real +# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including +# per-VMA Shared_Dirty inheritance and exclusion of post-fork VMAs. Real # Linux smaps exposes kernel-owned VMA/page accounting instead, so this is # intentionally not a reference-kernel invariant. diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 97a02e77..3563b22c 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -467,14 +467,21 @@ static int child_probe(uintptr_t target, size_t page_size, size_t stress_size) { + void *postfork = mmap(NULL, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (postfork == MAP_FAILED) + return 1; + char pid_path[64]; snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); const char *paths[] = {"/proc/self/smaps", pid_path}; for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { smaps_info_t info; - if (!load_smaps(paths[i], &info)) + if (!load_smaps(paths[i], &info)) { + munmap(postfork, page_size); return 1; + } bool ok = validate_layout(&info, target, stress, page_size, stress_size); const smaps_vma_t *first = find_vma(&info, target); @@ -482,15 +489,20 @@ static int child_probe(uintptr_t target, const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); const smaps_vma_t *stress_ro = find_vma(&info, stress); const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || - first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || - last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || - stress_rw->shared_dirty_kb == 0) { + !postfork_vma || first->shared_dirty_kb == 0 || + middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || + stress_ro->shared_dirty_kb != 0 || + stress_rw->shared_dirty_kb == 0 || + postfork_vma->shared_dirty_kb != 0) { free_smaps(&info); + munmap(postfork, page_size); return 1; } free_smaps(&info); } + munmap(postfork, page_size); return 0; } @@ -562,7 +574,7 @@ int main(void) EXPECT_TRUE(ok, "smaps parser/layout/completeness"); } - TEST("fork Shared_Dirty positive and read-only negative"); + TEST("fork Shared_Dirty inheritance and post-fork exclusion"); if (!fixture_ok) { FAIL("fixture unavailable"); } else { From 0fcc8f764004d9cbd3130650f933fe0f264a7bbf Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Sun, 2 Aug 2026 23:46:58 +0800 Subject: [PATCH 10/14] Extend fork-aware VMA tracking Preserve VMA lineage across fork-split mappings and keep synthetic smaps accounting consistent with per-VMA fork state. Add file-backed mremap regression coverage and wire the tests into the build and documentation. --- Makefile | 5 + docs/testing.md | 8 +- mk/tests.mk | 6 + src/core/guest.c | 143 ++++- src/core/guest.h | 42 +- src/syscall/mem.c | 967 +++++++++++++++++++++++------- tests/manifest.txt | 1 + tests/test-mremap-fork-tracking.c | 842 ++++++++++++++++++++++++++ tests/test-mremap-tail-emfile.c | 235 ++++++++ tests/test-proc-smap.c | 74 ++- 10 files changed, 2066 insertions(+), 257 deletions(-) create mode 100644 tests/test-mremap-fork-tracking.c create mode 100644 tests/test-mremap-tail-emfile.c diff --git a/Makefile b/Makefile index 5bea687c..2ce3816c 100644 --- a/Makefile +++ b/Makefile @@ -375,6 +375,11 @@ endif endif +## Build the libc-based file-backed mremap EMFILE regression probe +$(BUILD_DIR)/test-mremap-tail-emfile: tests/test-mremap-tail-emfile.c | $(BUILD_DIR) + @echo " CROSS $<" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< + include mk/tests.mk include mk/analysis.mk include mk/help.mk diff --git a/docs/testing.md b/docs/testing.md index c42e1b72..fb8a4cb4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -92,7 +92,8 @@ What they do: - the unit suite from `tests/manifest.txt` -- deliberately narrow: only tests that assert elfuse-internal implementation details with no real Linux counterpart (the EL1 shim fast-path suite, `test-mremap-infra`, - `test-oom-proc`), plus whatever `mk/tests.mk`'s `SANITIZER_SECTIONS` + `test-mremap-fork-tracking`, `test-oom-proc`), plus whatever + `mk/tests.mk`'s `SANITIZER_SECTIONS` needs for the `check-{asan,ubsan,tsan}` lanes. Everything that is meaningful to cross-check against a real Linux kernel lives exclusively in `tests/test-matrix.sh`'s `run_unit_tests` instead (see Test Matrix @@ -168,8 +169,9 @@ surface -- every binary that is meaningful to run against a real kernel, which is almost everything. It deliberately excludes only the handful of tests that assert elfuse-internal implementation details with no meaningful counterpart on a real kernel (the EL1 shim fast-path suite, `test-mremap-infra`, -`test-oom-proc` -- these live solely in `tests/manifest.txt` / `make check`, -see that file's header for the full split rationale). There is no separate +`test-mremap-fork-tracking`, `test-oom-proc` -- these live solely in +`tests/manifest.txt` / `make check`, see that file's header for the full split +rationale). There is no separate "core" vs "extended" test set inside the matrix; a test that has a real, understood divergence from the qemu reference kernel is listed in `QEMU_SKIP` with a comment explaining why instead -- see that variable in diff --git a/mk/tests.mk b/mk/tests.mk index c20e0a08..f879948d 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -16,6 +16,7 @@ test-vcpu-run-hooks-host test-identity-override-host \ test-dynamic-array-host \ test-string-builder-host \ + test-mremap-tail-emfile \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-fuse-alpine \ test-sysroot-nofollow test-sysroot-chdir test-sysroot-symlink-escape \ @@ -26,6 +27,11 @@ test-hello: $(ELFUSE_BIN) $(TEST_HELLO_DEP) @printf "$(BLUE)▸ Running$(RESET) test-hello\n" $(ELFUSE_BIN) $(TEST_DIR)/test-hello +## Run the libc-based file-backed region removal EMFILE regression probe +test-mremap-tail-emfile: $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + @printf "$(BLUE)▸ Running$(RESET) test-mremap-tail-emfile\n" + $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + ## Verify dispatch.tbl coverage of the kernel-supported syscall set check-syscall-coverage: @python3 scripts/check-syscall-coverage.py diff --git a/src/core/guest.c b/src/core/guest.c index 575d5d8d..7620dcb2 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1840,15 +1840,18 @@ int guest_get_used_regions(const guest_t *g, /* Semantic region tracking. * - * Check whether two adjacent regions can be merged. They must be contiguous in - * address space, have identical protection/flags/name, and have contiguous file - * offsets (so the merged region still represents valid mapping). For anonymous - * regions the offset is meaningless (always 0, but may become non-zero after - * split/trim), so the contiguity check is skipped. Without this, adjacent - * anonymous mmaps (common in megablock-style allocators) each create separate - * entries that exhaust the region table. + * Check whether two adjacent regions have merge-compatible layouts. An actual + * merge additionally requires a shared vma_id; a new same-generation mapping + * may adopt its compatible neighbor's ID below. Regions must be contiguous in + * address space, have identical protection/flags/name, and have contiguous + * file offsets (so the merged region still represents valid mapping). For + * anonymous regions the offset is meaningless (always 0, but may become + * non-zero after split/trim), so the contiguity check is skipped. Without this, + * adjacent anonymous mmaps (common in megablock-style allocators) each create + * separate entries that exhaust the region table. */ -static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +static bool regions_mergeable_layout(const guest_region_t *a, + const guest_region_t *b) { if (a->end != b->start) return false; @@ -1886,6 +1889,11 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return a->offset + (a->end - a->start) == b->offset; } +static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +{ + return a->vma_id == b->vma_id && regions_mergeable_layout(a, b); +} + /* First region whose start is >= start. regions[] is sorted by start. */ static int region_lower_bound_start(const guest_t *g, uint64_t start) { @@ -1983,7 +1991,8 @@ int guest_region_add_ex(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd, false); + offset, name, owned_backing_fd, false, + 0); } int guest_region_add_ex_gpa(guest_t *g, @@ -2004,7 +2013,31 @@ int guest_region_add_ex_gpa(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, gpa_base, prot, flags, - offset, name, owned_backing_fd, false); + offset, name, owned_backing_fd, false, + 0); +} + +static uint64_t allocate_vma_id(guest_t *g) +{ + uint64_t candidate = g->next_vma_id; + + for (;;) { + candidate++; + if (candidate == 0) + candidate = 1; + + bool in_use = false; + for (int i = 0; i < g->nregions; i++) { + if (g->regions[i].vma_id == candidate) { + in_use = true; + break; + } + } + if (!in_use) { + g->next_vma_id = candidate; + return candidate; + } + } } int guest_region_add_ex_owned(guest_t *g, @@ -2015,11 +2048,12 @@ int guest_region_add_ex_owned(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t vma_id) { return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, offset, name, owned_backing_fd, - inherited_at_fork); + inherited_at_fork, vma_id); } int guest_region_add_ex_owned_gpa(guest_t *g, @@ -2031,7 +2065,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t vma_id) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2044,6 +2079,12 @@ int guest_region_add_ex_owned_gpa(guest_t *g, return -1; } + bool new_vma = !vma_id; + if (new_vma) + vma_id = allocate_vma_id(g); + else if (vma_id > g->next_vma_id) + g->next_vma_id = vma_id; + /* Find insertion point (keep sorted by start address). */ int i = region_lower_bound_start(g, start); memmove(&g->regions[i + 1], &g->regions[i], @@ -2053,6 +2094,7 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->start = start; r->end = end; r->gpa_base = gpa_base; + r->vma_id = vma_id; r->prot = prot; r->flags = flags; r->offset = offset; @@ -2069,6 +2111,20 @@ int guest_region_add_ex_owned_gpa(guest_t *g, } g->nregions++; + /* Preserve the historical coalescing of compatible same-generation + * anonymous mmap calls: Linux may merge those into one VMA and the region + * tracker relies on that to stay below GUEST_MAX_REGIONS. Never adopt a + * neighbor's lineage across an inherited/private boundary, which is the + * provenance distinction find_mremap_source() must retain after fork. + */ + if (new_vma) { + if (i > 0 && regions_mergeable_layout(&g->regions[i - 1], r)) + r->vma_id = g->regions[i - 1].vma_id; + else if (i + 1 < g->nregions && + regions_mergeable_layout(r, &g->regions[i + 1])) + r->vma_id = g->regions[i + 1].vma_id; + } + /* Try to merge with adjacent regions to reduce table pressure. Merge right * first, then left (order matters: right merge does not change the index of * the left neighbor). @@ -2111,10 +2167,44 @@ int guest_preannounce(guest_t *g, return 0; } -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd) { + if (!reserved_backing_fd) + return -1; + *reserved_backing_fd = -1; if (end <= start) - return; + return 0; + + int first = guest_region_first_end_above(g, start); + for (int i = first; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->start < start && r->end > end) { + /* A full table follows the existing stale-tracker fallback and + * does not publish a right-hand record, so no fd is required. */ + if (g->nregions >= GUEST_MAX_REGIONS || r->backing_fd < 0) + return 0; + *reserved_backing_fd = dup(r->backing_fd); + return *reserved_backing_fd >= 0 ? 0 : -1; + } + } + return 0; +} + +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd) +{ + if (end <= start) { + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; + } /* In-place compaction: 'out' is the next output slot, 'in' is the next * input slot. Since the prefix [0, first) is untouched (it sorts strictly @@ -2174,18 +2264,16 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) right.gpa_base += trimmed; right.start = end; if (orig.backing_fd >= 0) { - right.backing_fd = dup(orig.backing_fd); - if (right.backing_fd < 0) - log_error( - "guest: dup() failed for region split " - "backing fd %d: %s", - orig.backing_fd, strerror(errno)); + right.backing_fd = reserved_backing_fd; + reserved_backing_fd = -1; } guest_region_clip_overlay(&right); g->regions[out + 1] = right; g->nregions = out + 2 + suffix_count; - return; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; } } @@ -2223,6 +2311,17 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) memmove(&g->regions[out], &g->regions[in], tail * sizeof(guest_region_t)); g->nregions = out + tail; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; +} + +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +{ + int reserved_backing_fd = -1; + if (guest_region_remove_prepare(g, start, end, &reserved_backing_fd) < 0) + return -1; + return guest_region_remove_reserved(g, start, end, reserved_backing_fd); } const guest_region_t *guest_region_find(const guest_t *g, uint64_t addr) diff --git a/src/core/guest.h b/src/core/guest.h index 65267cda..0645e70b 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -230,6 +230,12 @@ typedef struct { * identity-mapped regions; differs for high-VA guest * mappings whose VA and GPA diverge. */ + uint64_t vma_id; /* Stable logical-VMA lineage. Tracker splits and + * mremap moves preserve it; compatible same-generation + * mappings may share it when the tracker coalesces + * them. Unlike inherited_at_fork, this ID remains + * meaningful across subsequent forks. + */ int prot; /* LINUX_PROT_* flags */ int flags; /* LINUX_MAP_* flags (for /proc/self/maps display) */ uint64_t offset; /* File offset (for /proc/self/maps display) */ @@ -508,7 +514,8 @@ typedef struct { /* Semantic region tracking for munmap/mprotect/proc-self-maps */ guest_region_t regions[GUEST_MAX_REGIONS]; - int nregions; /* Number of active regions */ + int nregions; /* Number of active regions */ + uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */ /* Sticky flag set when guest_region_set_prot could not honor a request * because the region table was full. After this point the tracker no longer * faithfully reflects PTE state, so the mprotect fast path must fall back @@ -1182,8 +1189,8 @@ int guest_region_add_ex_gpa(guest_t *g, /* Like guest_region_add_ex, but consumes owned_backing_fd on success or * failure. inherited_at_fork identifies bytes copied from the latest fork - * snapshot; ordinary mmap/exec callers pass false, while fork/mremap restore - * paths preserve the source VMA's value. + * snapshot. vma_id preserves logical-VMA provenance across tracker splits and + * mremap moves; pass 0 for a new VMA. */ int guest_region_add_ex_owned(guest_t *g, uint64_t start, @@ -1193,7 +1200,8 @@ int guest_region_add_ex_owned(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork); + bool inherited_at_fork, + uint64_t vma_id); int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t start, uint64_t end, @@ -1203,7 +1211,8 @@ int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t offset, const char *name, int owned_backing_fd, - bool inherited_at_fork); + bool inherited_at_fork, + uint64_t vma_id); /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest @@ -1223,10 +1232,29 @@ int guest_preannounce(guest_t *g, uint64_t offset, const char *name); +/* Reserve any backing fd needed by an interior split in [start, end). + * The reservation must be consumed by guest_region_remove_reserved(). + * Returns 0 on success, -1 when the backing fd cannot be duplicated. + */ +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd); + /* Remove all region coverage in [start, end). Regions fully contained are - * deleted; partially overlapping regions are trimmed or split. + * deleted; partially overlapping regions are trimmed or split. Any required + * backing fd is reserved before the first metadata mutation. + */ +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end); + +/* Commit a removal using a backing fd reserved by + * guest_region_remove_prepare(). Ownership of reserved_backing_fd is consumed + * even when this call does not need an interior split. */ -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end); +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd); /* Find the region containing addr. * diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 8fc47d90..45943aa5 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -75,6 +75,7 @@ typedef struct { uint64_t start; uint64_t end; uint64_t gpa_base; + uint64_t vma_id; int prot; int flags; uint64_t offset; @@ -230,6 +231,168 @@ static void mark_region_backing_ro(guest_t *g, uint64_t start, uint64_t end) * remain unmarked. Keep the two portions as separate VMAs even when all other * metadata matches so the synthetic smaps view can distinguish them. */ +static bool mremap_backings_match(const guest_region_t *a, + const guest_region_t *b) +{ + if (a->backing_fd < 0 || b->backing_fd < 0) + return a->backing_fd < 0 && b->backing_fd < 0; + if (a->backing_fd == b->backing_fd) + return true; + + struct stat sa, sb; + if (fstat(a->backing_fd, &sa) < 0 || fstat(b->backing_fd, &sb) < 0) + return false; + return sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t gpa_base; + uint64_t offset; + int backing_fd; /* borrowed from the region tracker */ + bool overlay_active; + uint64_t overlay_start; + uint64_t overlay_end; +} mremap_source_segment_t; + +typedef struct { + const guest_region_t *first; + uint64_t inherited_prefix; + int nsegments; + mremap_source_segment_t *segments; +} mremap_source_t; + +static void dispose_mremap_source(mremap_source_t *source) +{ + if (!source) + return; + free(source->segments); + memset(source, 0, sizeof(*source)); +} + +static int64_t finish_mremap(mremap_source_t *source, int64_t result) +{ + dispose_mremap_source(source); + return result; +} + +/* Resolve a logical mremap source. Fork-aware growth intentionally leaves an + * inherited prefix and a child-private tail as separate records; those two + * records are still one VMA for mremap purposes. The stable vma_id proves that + * provenance even after another fork changes inherited_at_fork on both + * records. Reject any boundary with a different lineage so unrelated adjacent + * mappings cannot be copied as one source. + */ +static int find_mremap_source(const guest_t *g, + uint64_t start, + uint64_t size, + bool collect_segments, + mremap_source_t *source) +{ + uint64_t end = start + size; + const guest_region_t *first = guest_region_find(g, start); + if (!first) + return -LINUX_EFAULT; + + memset(source, 0, sizeof(*source)); + source->first = first; + + int index = (int) (first - g->regions); + if (index < 0 || index >= g->nregions) + goto invalid; + + uint64_t cursor = start; + uint64_t expected_gpa = first->gpa_base + (start - first->start); + uint64_t expected_offset = first->offset + (start - first->start); + uint64_t inherited_prefix = 0; + bool prefix_inherited = true; + int nsegments = 0; + for (int scan = index; scan < g->nregions && cursor < end; scan++) { + const guest_region_t *r = &g->regions[scan]; + if (r->start > cursor || r->end <= cursor) + goto invalid; + if (r != first) { + const guest_region_t *previous = &g->regions[scan - 1]; + if (previous->end != r->start || previous->prot != r->prot || + previous->flags != r->flags || previous->shared != r->shared || + previous->noreserve != r->noreserve || + previous->backing_ro != r->backing_ro || + strcmp(previous->name, r->name) != 0 || + !mremap_backings_match(previous, r)) + goto invalid; + if (!previous->vma_id || previous->vma_id != r->vma_id) + goto invalid; + } + + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + if (r->gpa_base + (cursor - r->start) != expected_gpa) + goto invalid; + if (!(r->flags & LINUX_MAP_ANONYMOUS) && + r->offset + (cursor - r->start) != expected_offset) + goto invalid; + + if (nsegments >= GUEST_MAX_REGIONS) + goto invalid; + nsegments++; + + if (prefix_inherited && r->inherited_at_fork) + inherited_prefix += segment_len; + else + prefix_inherited = false; + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + if (cursor != end) + goto invalid; + + /* Validation also gives the exact allocation size. Same-size non-fixed + * mremap stops here, so its no-op success cannot be turned into ENOMEM by + * segment bookkeeping. + */ + source->inherited_prefix = inherited_prefix; + source->nsegments = nsegments; + if (!collect_segments) + return 0; + + source->segments = + malloc((size_t) nsegments * sizeof(*source->segments)); + if (!source->segments) { + dispose_mremap_source(source); + return -LINUX_ENOMEM; + } + + cursor = start; + expected_gpa = first->gpa_base + (start - first->start); + expected_offset = first->offset + (start - first->start); + for (int segment_index = 0; segment_index < nsegments; segment_index++) { + const guest_region_t *r = &g->regions[index + segment_index]; + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + mremap_source_segment_t *segment = + &source->segments[segment_index]; + segment->start = cursor; + segment->end = segment_end; + segment->gpa_base = expected_gpa; + segment->offset = expected_offset; + segment->backing_fd = r->backing_fd; + segment->overlay_active = region_has_live_overlay(r); + segment->overlay_start = r->overlay_start; + segment->overlay_end = r->overlay_end; + + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + return 0; + +invalid: + dispose_mremap_source(source); + return -LINUX_EFAULT; +} + static int add_mremap_region(guest_t *g, uint64_t start, uint64_t old_size, @@ -239,36 +402,45 @@ static int add_mremap_region(guest_t *g, uint64_t offset, const char *name, int backing_fd, - bool inherited_at_fork) + bool inherited_at_fork, + uint64_t inherited_size, + int tail_backing_fd, + uint64_t vma_id) { - if (inherited_at_fork && old_size > 0 && old_size < new_size) { - int tail_backing_fd = -1; - if (backing_fd >= 0) { - tail_backing_fd = dup(backing_fd); - if (tail_backing_fd < 0) { - close(backing_fd); - return -1; - } + if (!inherited_at_fork) + inherited_size = 0; + if (inherited_size > old_size) + inherited_size = old_size; + if (inherited_size > new_size) + inherited_size = new_size; + + if (inherited_size > 0 && inherited_size < new_size) { + if (backing_fd >= 0 && tail_backing_fd < 0) { + close(backing_fd); + return -1; } - if (guest_region_add_ex_owned(g, start, start + old_size, prot, flags, - offset, name, backing_fd, true) < 0) { + if (guest_region_add_ex_owned(g, start, start + inherited_size, prot, + flags, offset, name, backing_fd, true, + vma_id) < 0) { if (tail_backing_fd >= 0) close(tail_backing_fd); return -1; } - - if (guest_region_add_ex_owned(g, start + old_size, start + new_size, - prot, flags, offset + old_size, name, - tail_backing_fd, false) < 0) { - guest_region_remove(g, start, start + old_size); + if (guest_region_add_ex_owned(g, start + inherited_size, + start + new_size, prot, flags, + offset + inherited_size, name, + tail_backing_fd, false, vma_id) < 0) { + guest_region_remove(g, start, start + inherited_size); return -1; } return 0; } - return guest_region_add_ex_owned(g, start, start + new_size, prot, flags, - offset, name, backing_fd, - inherited_at_fork && old_size > 0); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return guest_region_add_ex_owned( + g, start, start + new_size, prot, flags, offset, name, backing_fd, + inherited_size == new_size && inherited_size > 0, vma_id); } static void region_clip_overlay(guest_region_t *r) @@ -290,43 +462,51 @@ static void region_clip_overlay(guest_region_t *r) region_clear_overlay(r); } -static void split_regions_at_boundary(guest_t *g, uint64_t boundary) +static int split_regions_at_boundary(guest_t *g, uint64_t boundary) { if (boundary == 0) - return; + return 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (boundary <= r->start) break; if (boundary >= r->end) continue; if (g->nregions >= GUEST_MAX_REGIONS) { log_error( - "guest: region table full, cleanup split skipped at " + "guest: region table full, region split skipped at " "0x%llx", (unsigned long long) boundary); - return; + return -LINUX_ENOMEM; } + guest_region_t left = *r; + guest_region_t right = *r; + if (right.backing_fd >= 0) { + right.backing_fd = dup(right.backing_fd); + if (right.backing_fd < 0) { + log_error("guest: dup() failed for region split: %s", + strerror(errno)); + return -LINUX_ENOMEM; + } + } + + left.end = boundary; + right.offset += boundary - right.start; + right.gpa_base += boundary - right.start; + right.start = boundary; + region_clip_overlay(&left); + region_clip_overlay(&right); + memmove(&g->regions[i + 1], &g->regions[i], (g->nregions - i) * sizeof(guest_region_t)); + g->regions[i] = left; + g->regions[i + 1] = right; g->nregions++; - - g->regions[i].end = boundary; - g->regions[i + 1].offset += (boundary - g->regions[i + 1].start); - g->regions[i + 1].gpa_base += (boundary - g->regions[i + 1].start); - g->regions[i + 1].start = boundary; - if (g->regions[i + 1].backing_fd >= 0) { - g->regions[i + 1].backing_fd = dup(g->regions[i + 1].backing_fd); - if (g->regions[i + 1].backing_fd < 0) - log_error("guest: dup() failed for cleanup split: %s", - strerror(errno)); - } - region_clip_overlay(&g->regions[i]); - region_clip_overlay(&g->regions[i + 1]); - return; + return 0; } + return 0; } static uint64_t find_free_gap_inner(const guest_t *g, @@ -901,7 +1081,7 @@ static int64_t sys_mmap_high_va(guest_t *g, } if (guest_region_add_ex_owned_gpa(g, addr, addr + length, gpa_base, prot, flags, offset, NULL, track_backing_fd, - false) < 0) + false, 0) < 0) goto fail; /* Ownership of track_backing_fd is now held by the new region. The fail * handler below skips closing when track_backing_fd < 0, so subsequent @@ -1101,6 +1281,167 @@ static int restore_file_overlay_range(guest_t *g, return 0; } +static bool mremap_source_has_overlay(const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + if (source->segments[i].overlay_active) + return true; + } + return false; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t offset; + int backing_fd; + bool active; + uint64_t overlay_start; + uint64_t overlay_end; +} saved_overlay_t; + +typedef void (*saved_overlay_getter_t)(const void *saved, + int index, + saved_overlay_t *overlay); + +static uint64_t saved_overlay_file_offset(const saved_overlay_t *overlay) +{ + if (overlay->overlay_start >= overlay->start) + return overlay->offset + (overlay->overlay_start - overlay->start); + return overlay->offset - (overlay->start - overlay->overlay_start); +} + +/* Reinstall each distinct host overlay once, then restore metadata on every + * tracker fragment that shared it. cleanup_overlays_in_range() can fail after + * removing only a subset, so replaying the complete saved set is rollback-safe. + */ +static int restore_saved_overlays_in_place(guest_t *g, + const void *saved, + int n, + saved_overlay_getter_t get) +{ + for (int i = 0; i < n; i++) { + saved_overlay_t overlay; + get(saved, i, &overlay); + if (!overlay.active || overlay.backing_fd < 0) + continue; + + uint64_t file_off = saved_overlay_file_offset(&overlay); + bool first = true; + for (int j = 0; j < i; j++) { + saved_overlay_t previous; + get(saved, j, &previous); + if (previous.active && previous.backing_fd >= 0 && + previous.overlay_start == overlay.overlay_start && + previous.overlay_end == overlay.overlay_end && + saved_overlay_file_offset(&previous) == file_off) { + first = false; + break; + } + } + + if (first) { + int err = restore_file_overlay_range( + g, overlay.start, overlay.end, overlay.overlay_start, + overlay.overlay_end, overlay.backing_fd, file_off); + if (err < 0) + return err; + } else { + mark_overlay_metadata_range(g, overlay.start, overlay.end, + overlay.overlay_start, + overlay.overlay_end); + } + } + return 0; +} + +static void get_mremap_source_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const mremap_source_t *source = saved; + const mremap_source_segment_t *segment = &source->segments[index]; + *overlay = (saved_overlay_t) { + .start = segment->start, + .end = segment->end, + .offset = segment->offset, + .backing_fd = segment->backing_fd, + .active = segment->overlay_active, + .overlay_start = segment->overlay_start, + .overlay_end = segment->overlay_end, + }; +} + +static int restore_mremap_source_overlays_in_place( + guest_t *g, + const mremap_source_t *source) +{ + return restore_saved_overlays_in_place(g, source, source->nsegments, + get_mremap_source_overlay); +} + +/* The host overlay stays installed during an in-place growth; only the region + * records are replaced. Reapply the saved metadata to the corresponding new + * records without remapping the host VA. + */ +static void mark_mremap_source_overlay_metadata(guest_t *g, + const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + if (segment->overlay_active) + mark_overlay_metadata_range(g, segment->start, segment->end, + segment->overlay_start, + segment->overlay_end); + } +} + +/* Copy each source segment according to its own backing state. Live-overlay + * bytes must be refreshed from the file after the overlay is removed; private + * fork-grown bytes remain in the slab and must be copied from their GPA. + */ +static int copy_mremap_source(guest_t *g, + uint64_t dest_gpa, + uint64_t source_start, + uint64_t length, + const mremap_source_t *source) +{ + uint64_t source_end = source_start + length; + uint64_t cursor = source_start; + + for (int i = 0; i < source->nsegments && cursor < source_end; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + uint64_t start = segment->start > cursor ? segment->start : cursor; + uint64_t end = segment->end < source_end ? segment->end : source_end; + if (end <= start) + continue; + if (start != cursor) + return -LINUX_EFAULT; + + uint64_t len = end - start; + uint64_t dest = dest_gpa + (start - source_start); + if (segment->overlay_active) { + if (segment->backing_fd < 0) + return -LINUX_EFAULT; + int err = read_file_range_to_guest( + g, dest, segment->backing_fd, + segment->offset + (start - segment->start), len); + if (err < 0) + return err; + } else { + uint8_t *dest_ptr = host_ptr_for_gpa(g, dest); + uint8_t *source_ptr = host_ptr_for_gpa( + g, segment->gpa_base + (start - segment->start)); + if (!dest_ptr || !source_ptr) + return -LINUX_EFAULT; + memmove(dest_ptr, source_ptr, len); + } + cursor = end; + } + + return cursor == source_end ? 0 : -LINUX_EFAULT; +} + typedef struct { uint64_t overlay_start; uint64_t overlay_len; @@ -1148,8 +1489,12 @@ static int capture_region_snapshots(guest_t *g, region_snapshot_t *snaps, int max_snaps) { - split_regions_at_boundary(g, start); - split_regions_at_boundary(g, end); + int split_err = split_regions_at_boundary(g, start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, end); + if (split_err < 0) + return split_err; int n = 0; for (int i = 0; i < g->nregions; i++) { @@ -1167,6 +1512,7 @@ static int capture_region_snapshots(guest_t *g, snap->start = r->start; snap->end = r->end; snap->gpa_base = r->gpa_base; + snap->vma_id = r->vma_id; snap->prot = r->prot; snap->flags = r->flags; snap->offset = r->offset; @@ -1189,48 +1535,58 @@ static int capture_region_snapshots(guest_t *g, return n; } -static int restore_snapshot_overlays_in_place(guest_t *g, - const region_snapshot_t *snaps, - int n) +/* MREMAP_FIXED may remove a destination fragment that used to share the same + * tracker backing fd as a source fragment. Rebind file-backed source segments + * to the owned source snapshots before destination removal, so later overlay + * restore and pread-based copies cannot observe a closed borrowed fd. */ +static int rebind_mremap_source_backings(mremap_source_t *source, + const region_snapshot_t *snaps, + int n) { - for (int i = 0; i < n; i++) { - const region_snapshot_t *snap = &snaps[i]; - if (!snap->overlay_active || snap->backing_fd < 0) + for (int i = 0; i < source->nsegments; i++) { + mremap_source_segment_t *segment = &source->segments[i]; + if (segment->backing_fd < 0) continue; - bool first = true; - uint64_t snap_file_off = - snap->offset + (snap->overlay_start - snap->start); - for (int j = 0; j < i; j++) { - const region_snapshot_t *prev = &snaps[j]; - if (!prev->overlay_active || prev->backing_fd < 0) - continue; - uint64_t prev_file_off = - prev->offset + (prev->overlay_start - prev->start); - if (prev->overlay_start == snap->overlay_start && - prev->overlay_end == snap->overlay_end && - prev_file_off == snap_file_off) { - first = false; + int stable_fd = -1; + for (int j = 0; j < n; j++) { + if (snaps[j].start <= segment->start && + segment->start < snaps[j].end && snaps[j].backing_fd >= 0) { + stable_fd = snaps[j].backing_fd; break; } } - - if (first) { - int err = restore_file_overlay_range( - g, snap->start, snap->end, snap->overlay_start, - snap->overlay_end, snap->backing_fd, snap_file_off); - if (err < 0) - return err; - continue; - } - - mark_overlay_metadata_range(g, snap->start, snap->end, - snap->overlay_start, snap->overlay_end); + if (stable_fd < 0) + return -LINUX_EFAULT; + segment->backing_fd = stable_fd; } - return 0; } +static void get_region_snapshot_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const region_snapshot_t *snap = &((const region_snapshot_t *) saved)[index]; + *overlay = (saved_overlay_t) { + .start = snap->start, + .end = snap->end, + .offset = snap->offset, + .backing_fd = snap->backing_fd, + .active = snap->overlay_active, + .overlay_start = snap->overlay_start, + .overlay_end = snap->overlay_end, + }; +} + +static int restore_snapshot_overlays_in_place(guest_t *g, + const region_snapshot_t *snaps, + int n) +{ + return restore_saved_overlays_in_place(g, snaps, n, + get_region_snapshot_overlay); +} + static bool snapshot_has_materialized_ptes(const region_snapshot_t *snap) { return snap->prot != LINUX_PROT_NONE && @@ -1289,7 +1645,7 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) if (guest_region_add_ex_owned_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd, snap->inherited_at_fork) < 0) { + snap->backing_fd, snap->inherited_at_fork, snap->vma_id) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -1768,8 +2124,12 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) uint64_t host_start = ALIGN_DOWN(start, hps); uint64_t host_end = ALIGN_UP(end, hps); - split_regions_at_boundary(g, host_start); - split_regions_at_boundary(g, host_end); + int split_err = split_regions_at_boundary(g, host_start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, host_end); + if (split_err < 0) + return split_err; /* Snapshot affected ranges first; the host-side mmap calls below do not * touch the region array, but a future caller invariant is to allow this @@ -1826,6 +2186,22 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) /* Memory syscalls (tightly coupled to guest.h). */ +static bool heap_tail_can_extend(const guest_region_t *tail, + const guest_region_t *heap, + uint64_t old_brk) +{ + const int heap_flags = LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS; + + return tail->start == heap->end && tail->end == old_brk && + tail->end > tail->start && tail->gpa_base == tail->start && + tail->vma_id == heap->vma_id && + tail->prot == (LINUX_PROT_READ | LINUX_PROT_WRITE) && + tail->flags == heap_flags && tail->offset == 0 && + tail->backing_fd < 0 && !tail->shared && !tail->noreserve && + !tail->backing_ro && !tail->inherited_at_fork && + !region_has_live_overlay(tail) && !strcmp(tail->name, "[heap]"); +} + int64_t sys_brk(guest_t *g, uint64_t addr) { /* brk addresses as seen by the guest are IPA-based */ @@ -1847,6 +2223,16 @@ int64_t sys_brk(guest_t *g, uint64_t addr) return (int64_t) ipa_brk; } + /* Shrinking can split a file-backed semantic region when a fork child has + * grown and reshaped its heap metadata. Reserve the right-hand region's + * backing fd before publishing the new break, so EMFILE leaves both the + * break and region table unchanged. + */ + int shrink_remove_fd = -1; + if (new_off < old_brk && + guest_region_remove_prepare(g, new_off, old_brk, &shrink_remove_fd) < 0) + return (int64_t) ipa_brk; + /* Materialize any newly exposed heap pages. This must handle both: * 1. growth into brand-new 2 MiB blocks, and * 2. growth within an already-split block where finalize_block_perms() @@ -1874,7 +2260,15 @@ int64_t sys_brk(guest_t *g, uint64_t addr) * avoids the remove+add gap where a concurrent /proc/self/maps reader could * see no heap region. */ - if (new_off > g->brk_base) { + if (new_off < old_brk) { + /* Trim every semantic heap segment covered by the released suffix. + * In a fork child this may shorten or remove the private tail while + * leaving the inherited prefix intact. Keeping the tracker end equal + * to brk_current prevents a later regrowth from overlapping stale + * tail metadata. + */ + guest_region_remove_reserved(g, new_off, old_brk, shrink_remove_fd); + } else if (new_off > g->brk_base) { bool found = false; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start == g->brk_base && @@ -1883,17 +2277,31 @@ int64_t sys_brk(guest_t *g, uint64_t addr) uint64_t old_heap_end = heap->end; if (new_off > old_heap_end && heap->inherited_at_fork) { /* Keep the fork-snapshot portion separate from pages - * materialized by this post-fork brk growth. If the - * semantic table is full, retain the historical - * conservative range and mark it stale rather than - * failing an otherwise successful brk syscall. + * materialized by post-fork brk growth. On later growths, + * extend the existing child-private tail rather than + * adding an overlapping range from the old boundary. */ - if (guest_region_add( - g, old_heap_end, new_off, - LINUX_PROT_READ | LINUX_PROT_WRITE, - LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, - "[heap]") < 0) { - heap->end = new_off; + guest_region_t *right = + i + 1 < g->nregions ? &g->regions[i + 1] : NULL; + guest_region_t *tail = + right && heap_tail_can_extend(right, heap, old_brk) + ? right + : NULL; + if (tail) { + if (new_off > tail->end) + tail->end = new_off; + } else if (new_off > old_brk && + guest_region_add_ex_owned( + g, old_brk, new_off, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, + "[heap]", -1, false, heap->vma_id) < 0) { + /* Widening the inherited prefix would either overlap + * an incompatible child-private tail or mislabel new + * pages as inherited. Keep the original boundary; brk + * memory already grew successfully, so only the + * semantic tracker becomes stale. + */ g->regions_tracker_stale = true; } } else { @@ -2628,7 +3036,7 @@ int64_t sys_mmap(guest_t *g, */ if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, track_backing_fd, false) < 0) { + NULL, track_backing_fd, false, 0) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -2758,12 +3166,20 @@ int64_t sys_mremap(guest_t *g, if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; - /* Verify the whole source range is covered by one tracked VMA. mremap() - * must not copy holes or unrelated adjacent mappings. + /* Verify the whole source range is covered by one logical VMA. A + * fork-aware growth can split that VMA at the inherited/private boundary, + * but no unrelated adjacent mapping may be included. */ - const guest_region_t *src_reg = guest_region_find(g, old_off); - if (!src_reg || src_reg->end - old_off < old_size) - return -LINUX_EFAULT; + mremap_source_t source; + bool collect_source_segments = + old_size != new_size || (flags & LINUX_MREMAP_FIXED); + int source_err = find_mremap_source(g, old_off, old_size, + collect_source_segments, &source); + if (source_err < 0) + return source_err; + const guest_region_t *src_reg = source.first; + uint64_t source_inherited_size = source.inherited_prefix; + uint64_t source_vma_id = src_reg->vma_id; /* Capture the source region's GPA layout before any region mutation below * invalidates src_reg. src_gpa_base + (va_off - src_start) is the backing @@ -2776,41 +3192,48 @@ int64_t sys_mremap(guest_t *g, /* Same size: nothing to do */ if (old_size == new_size && !(flags & LINUX_MREMAP_FIXED)) - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); /* Shrinking mremap keeps the base address and releases only the tail. */ if (new_size < old_size && !(flags & LINUX_MREMAP_FIXED)) { uint64_t tail_off = old_off + new_size, tail_end = old_off + old_size; + int tail_remove_fd = -1; + if (guest_region_remove_prepare(g, tail_off, tail_end, + &tail_remove_fd) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); /* Restore slab backing under any tail overlay before zeroing so the * memset does not write zeros into a file. */ int cleanup_err = cleanup_overlays_in_range(g, tail_off, tail_end); - if (cleanup_err < 0) - return cleanup_err; + if (cleanup_err < 0) { + if (tail_remove_fd >= 0) + close(tail_remove_fd); + return finish_mremap(&source, cleanup_err); + } /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, tail_end - tail_off); - guest_region_remove(g, tail_off, tail_end); + guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) g->mmap_rx_gap_hint = tail_off; - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } /* MREMAP_FIXED: move to a specific new address */ if (flags & LINUX_MREMAP_FIXED) { if (new_addr & 4095) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); uint64_t new_off = new_addr - g->ipa_base; /* MREMAP_FIXED dest stays primary-only for the same reason as the * source check above. */ if (new_off > g->guest_size || new_size > g->guest_size - new_off) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Same infrastructure protection as the source range: the move tail * removes any existing dest region and rewrites PTEs, which would @@ -2818,12 +3241,12 @@ int64_t sys_mremap(guest_t *g, * infra. */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; if (old_off < new_end && new_off < old_end) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); remove_range_t removed[] = { {old_off, old_end}, @@ -2835,27 +3258,38 @@ int64_t sys_mremap(guest_t *g, * metadata. The overlap check above prevents this case, but capturing * first is still the safe ordering. */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; int added_regions = - source_inherited_at_fork && old_size < new_size ? 2 : 1; + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; if (!region_has_capacity_after_removes(g, removed, 2, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; char track_name[sizeof(old_reg->name)] = {0}; /* Heap-allocated to avoid blowing the ~512 KiB default macOS thread * stack: each region_snapshot_t array is GUEST_MAX_REGIONS * @@ -2876,7 +3310,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } source_nsnaps = capture_region_snapshots( @@ -2886,7 +3322,20 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return source_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, source_nsnaps); + } + int rebind_err = + rebind_mremap_source_backings(&source, source_snaps, source_nsnaps); + if (rebind_err < 0) { + dispose_region_snapshots(&source_snaps, &source_nsnaps); + free(dest_snaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, rebind_err); } dest_nsnaps = capture_region_snapshots(g, new_off, new_off + new_size, dest_snaps, GUEST_MAX_REGIONS); @@ -2895,7 +3344,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return dest_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, dest_nsnaps); } if (source_overlay) { @@ -2908,7 +3359,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } } @@ -2922,7 +3375,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2930,7 +3385,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { @@ -2941,7 +3398,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2949,7 +3408,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } /* Remove existing mappings at the destination after all fallible @@ -2957,19 +3418,19 @@ int64_t sys_mremap(guest_t *g, */ guest_region_remove(g, new_off, new_off + new_size); - /* Copy data (use memmove for potential overlap). If the source has a - * live overlay, the read side of the memmove pulls live file content; - * the destination receives a private snapshot at mremap time (no + /* Copy each logical source segment according to its own backing state. + * The destination receives a private snapshot at mremap time (no * overlay reapplied), and msync's emulated pwrite-the-diff path keeps * subsequent writes consistent. */ uint64_t copy_len = old_size < new_size ? old_size : new_size; if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, copy_len); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, copy_len, &source); if (copy_err < 0) { int restore_err = restore_snapshot_overlays_in_place( g, source_snaps, source_nsnaps); @@ -2990,21 +3451,14 @@ int64_t sys_mremap(guest_t *g, restore_err = pt_err; if (track_backing_fd >= 0) close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); if (restore_err < 0) - return restore_err; - return copy_err; + return finish_mremap(&source, restore_err); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA (identity for primary sources, - * overflow/mapping backing for high-VA). The destination is always - * a fresh primary-window range, so it never overlaps the source and - * the copy direction does not matter. - */ - memmove((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - copy_len); } /* Zero any extension beyond old data */ if (new_size > old_size) @@ -3025,17 +3479,19 @@ int64_t sys_mremap(guest_t *g, if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, source_inherited_at_fork) < 0) { + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) { (void) restore_region_snapshots(g, dest_snaps, dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Grow in place: try to extend without moving */ @@ -3048,7 +3504,7 @@ int64_t sys_mremap(guest_t *g, * it. */ if (guest_range_hits_infra(g, grow_off, grow_off + grow_len)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Check if the space after the old region is free (overflow-safe) */ if (grow_off <= g->guest_size && grow_len <= g->guest_size - grow_off) { @@ -3069,31 +3525,54 @@ int64_t sys_mremap(guest_t *g, if (can_grow) { remove_range_t removed = {old_off, old_off + old_size}; /* Extend in place */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); - bool old_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t old_overlay_start = - old_overlay ? old_reg->overlay_start : 0; - uint64_t old_overlay_end = - old_overlay ? old_reg->overlay_end : 0; + int tail_backing_fd = -1; + if (source_inherited_size > 0 && + source_inherited_size < new_size && track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } bool old_backing_ro = old_reg && old_reg->backing_ro; bool old_inherited_at_fork = old_reg && old_reg->inherited_at_fork; - int added_regions = old_inherited_at_fork ? 2 : 1; + int added_regions = source_inherited_size > 0 && + source_inherited_size < new_size + ? 2 + : 1; if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + if (old_reg && old_reg->backing_fd >= 0 && + track_backing_fd < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, @@ -3102,22 +3581,26 @@ int64_t sys_mremap(guest_t *g, if (mremap_extend_range(g, grow_off, grow_len, prot) < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); /* Update region tracking: remove old, add extended */ - guest_region_remove(g, old_off, old_off + old_size); - if (add_mremap_region( - g, old_off, old_size, new_size, prot, track_flags, - track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, old_inherited_at_fork) < 0) - return -LINUX_ENOMEM; - if (old_overlay) - mark_overlay_metadata_range(g, old_off, old_off + old_size, - old_overlay_start, - old_overlay_end); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); + if (add_mremap_region(g, old_off, old_size, new_size, prot, + track_flags, track_offset, + track_name[0] ? track_name : NULL, + track_backing_fd, old_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + mark_mremap_source_overlay_metadata(g, &source); if (old_backing_ro) mark_region_backing_ro(g, old_off, old_off + new_size); @@ -3131,36 +3614,37 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } } /* Growth in place failed; MREMAP_MAYMOVE is required */ if (!(flags & LINUX_MREMAP_MAYMOVE)) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Allocate a new region and move */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t source_overlay_start = - source_overlay ? old_reg->overlay_start : 0; - uint64_t source_overlay_end = source_overlay ? old_reg->overlay_end : 0; + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; - uint64_t source_overlay_file_off = - source_overlay - ? old_reg->offset + (source_overlay_start - old_reg->start) - : 0; char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, sizeof(track_name)); @@ -3182,56 +3666,85 @@ int64_t sys_mremap(guest_t *g, if (new_off == UINT64_MAX) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } remove_range_t removed = {old_off, old_off + old_size}; int added_regions = - source_inherited_at_fork && old_size < new_size ? 2 : 1; + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_overlay) { int cleanup_err = cleanup_overlays_in_range(g, old_off, old_off + old_size); if (cleanup_err < 0) { + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + if (restore_err < 0) + return finish_mremap(&source, restore_err); + return finish_mremap(&source, cleanup_err); } } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { if (source_overlay) { - int restore_err = restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (restore_err < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, restore_err); } } if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - /* Copy old data, zero extension. The new range was just allocated from - * a free gap so it has no overlays to clean up; the source may have an - * overlay, which is read transparently by the memcpy before its - * underlying slab is restored below. + /* Copy each source segment according to its own backing state, then + * zero the extension. The new range is a fresh gap and receives no + * live overlay. */ if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, old_size); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, old_size, &source); if (copy_err < 0) { /* Roll back both sides: re-apply the source overlay so the * caller's MAP_SHARED is not silently demoted to a slab @@ -3239,23 +3752,16 @@ int64_t sys_mremap(guest_t *g, * allocated via mremap_extend_range so the guest does not see * phantom zero pages where the failed mremap landed. */ - (void) restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + (void) restore_mremap_source_overlays_in_place(g, &source); guest_invalidate_ptes(g, new_off, new_off + new_size); if (track_backing_fd >= 0) close(track_backing_fd); - return copy_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA so high-VA sources copy from - * their real backing (identity for primary: == host_base + - * old_off). The destination is a fresh primary-window gap. - */ - memcpy((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - old_size); } memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); @@ -3265,7 +3771,8 @@ int64_t sys_mremap(guest_t *g, */ memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, old_size); - guest_region_remove(g, old_off, old_off + old_size); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; @@ -3275,8 +3782,10 @@ int64_t sys_mremap(guest_t *g, /* Track new region */ if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, track_offset, track_name[0] ? track_name : NULL, - track_backing_fd, source_inherited_at_fork) < 0) - return -LINUX_ENOMEM; + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); @@ -3290,11 +3799,11 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Should not reach here */ - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); } /* sys_madvise. */ @@ -3496,20 +4005,34 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + /* An interior removal from a file-backed region needs a second owned fd + * for the surviving right half. Reserve it before changing overlays, + * page tables, or host memory so descriptor exhaustion is failure-atomic. + */ + int remove_fd = -1; + if (guest_region_remove_prepare(g, unmap_off, end, &remove_fd) < 0) + return -LINUX_ENOMEM; + /* Restore slab backing under any active MAP_SHARED file overlay before * zeroing the host VA. Without this, the memset below would write zeros * directly into the file. */ int cleanup_err = cleanup_overlays_in_range(g, unmap_off, end); - if (cleanup_err < 0) + if (cleanup_err < 0) { + if (remove_fd >= 0) + close(remove_fd); return cleanup_err; + } /* Invalidate PTEs first. This may need to split a 2MiB block which can fail * if the page table pool is exhausted. Failing before region removal keeps * metadata consistent. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); return -LINUX_ENOMEM; + } for (int i = 0; i < g->nregions; i++) { guest_region_t *r = &g->regions[i]; if (r->start >= end) @@ -3522,7 +4045,7 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) uint64_t zend = (r->end < end) ? r->end : end; memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); } - guest_region_remove(g, unmap_off, end); + guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = unmap_off; if (unmap_off < g->mmap_rx_gap_hint) @@ -3576,9 +4099,17 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { if (region_range_overlaps(g, addr, addr + length)) { - if (guest_invalidate_ptes(g, addr, addr + length) < 0) + int remove_fd = -1; + if (guest_region_remove_prepare(g, addr, addr + length, + &remove_fd) < 0) return -LINUX_ENOMEM; - guest_region_remove(g, addr, addr + length); + if (guest_invalidate_ptes(g, addr, addr + length) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + guest_region_remove_reserved(g, addr, addr + length, + remove_fd); } return 0; } diff --git a/tests/manifest.txt b/tests/manifest.txt index d33d9527..552a01e0 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -109,6 +109,7 @@ test-mmap-hint [section] mremap tests test-mremap test-mremap-infra +test-mremap-fork-tracking test-shim-cred-race [section] msync MAP_SHARED tests diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c new file mode 100644 index 00000000..0bdf9b31 --- /dev/null +++ b/tests/test-mremap-fork-tracking.c @@ -0,0 +1,842 @@ +/* + * elfuse-internal mremap fork-tracking tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * These cases exercise elfuse's inherited-at-fork region bookkeeping, not a + * portable Linux ABI contract. Do not add this binary to test-matrix.sh: + * Linux can merge the adjacent file mappings below, and extends MAP_SHARED + * mappings directly from the backing file rather than creating elfuse's + * child-private tracking tail. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif +#ifndef MREMAP_FIXED +#define MREMAP_FIXED 2 +#endif + +static bool set_program_break(uintptr_t address) +{ + return brk((void *) address) == 0 && sbrk(0) == (void *) address; +} + +static ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) +{ + if (bufsz == 0) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + ssize_t total = 0; + while ((size_t) total < bufsz - 1) { + ssize_t n = read(fd, buf + total, bufsz - 1 - (size_t) total); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + int saved_errno = errno; + (void) close(fd); + errno = saved_errno; + return -1; + } + if (n == 0) + break; + total += n; + } + buf[total] = '\0'; + (void) close(fd); + return total; +} + +static void *reserve_then_map_fixed(size_t reserve_length, + size_t mapping_length, + int prot, + int flags, + int fd, + off_t offset) +{ + void *base = mmap(NULL, reserve_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) + return MAP_FAILED; + if (munmap(base, reserve_length) != 0) { + int saved_errno = errno; + (void) munmap(base, reserve_length); + errno = saved_errno; + return MAP_FAILED; + } + return mmap(base, mapping_length, prot, flags | MAP_FIXED, fd, offset); +} + +static bool parse_maps_hex(const char **cursor, + const char *end, + char delimiter, + uintptr_t *value) +{ + const char *start = *cursor; + uintptr_t parsed = 0; + while (*cursor < end && **cursor != delimiter) { + unsigned int digit; + char c = **cursor; + if (c >= '0' && c <= '9') + digit = (unsigned int) (c - '0'); + else if (c >= 'a' && c <= 'f') + digit = (unsigned int) (c - 'a' + 10); + else + return false; + parsed = parsed * 16 + digit; + (*cursor)++; + } + if (*cursor == start || *cursor >= end) + return false; + *value = parsed; + return true; +} + +static bool mapping_is_rw_private(uintptr_t address) +{ + char maps[64 * 1024]; + ssize_t length = read_file_nul("/proc/self/maps", maps, sizeof(maps)); + if (length <= 0) + return false; + + const char *cursor = maps; + const char *end = maps + length; + while (cursor < end) { + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&cursor, end, '-', &start)) + return false; + cursor++; + if (!parse_maps_hex(&cursor, end, ' ', &limit)) + return false; + cursor++; + if (cursor + 4 > end) + return false; + if (start <= address && address < limit) { + return cursor[0] == 'r' && cursor[1] == 'w' && cursor[2] == '-' && + cursor[3] == 'p'; + } + while (cursor < end && *cursor != '\n') + cursor++; + if (cursor < end) + cursor++; + } + return false; +} + +static char maps_buffer[1024 * 1024]; + +static bool read_maps_snapshot(ssize_t *length_out) +{ + ssize_t length = + read_file_nul("/proc/self/maps", maps_buffer, sizeof(maps_buffer)); + if (length <= 0 || (size_t) length == sizeof(maps_buffer) - 1) + return false; + *length_out = length; + return true; +} + +static bool count_maps_entries(int *count_out) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + int count = 0; + for (ssize_t i = 0; i < length; i++) { + if (maps_buffer[i] == '\n') + count++; + } + *count_out = count; + return true; +} + +static bool heap_headers_do_not_overlap(int *heap_count) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + const char heap_name[] = "[heap]"; + const char *cursor = maps_buffer; + const char *end = maps_buffer + length; + uintptr_t previous_end = 0; + int count = 0; + + while (cursor < end) { + const char *line = cursor; + while (cursor < end && *cursor != '\n') + cursor++; + const char *line_end = cursor; + if (cursor < end) + cursor++; + + bool is_heap = false; + for (const char *p = line; p + sizeof(heap_name) - 1 <= line_end; p++) { + if (memcmp(p, heap_name, sizeof(heap_name) - 1) == 0) { + is_heap = true; + break; + } + } + if (!is_heap) + continue; + + const char *field = line; + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&field, line_end, '-', &start)) + return false; + field++; + if (!parse_maps_hex(&field, line_end, ' ', &limit)) + return false; + if (count > 0 && start < previous_end) + return false; + previous_end = limit; + count++; + } + + *heap_count = count; + return true; +} + +static void test_postfork_adjacent_anon_rejected(void) +{ + TEST("mremap rejects unrelated post-fork tail"); + + const size_t span = 64 * 1024; + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (first == MAP_FAILED) { + FAIL("inherited anonymous mmap failed"); + return; + } + void *base = first; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(first, span); + return; + } + if (pid == 0) { + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (second != (char *) base + span) + _exit(30); + + errno = 0; + void *shrunk = mremap(base, 2 * span, span, 0); + if (shrunk == MAP_FAILED && errno == EFAULT) + _exit(0); + _exit(31); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("mremap accepted unrelated post-fork mapping"); + + munmap(first, span); +} + +static void test_adjacent_file_vmas_rejected(void) +{ + TEST("mremap rejects adjacent independent file VMAs"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-adjacent-XXXXXX"; + int fd1 = mkstemp(tmpl); + if (fd1 < 0) { + FAIL("mkstemp failed"); + return; + } + int fd2 = open(tmpl, O_RDWR); + unlink(tmpl); + if (fd2 < 0 || ftruncate(fd1, (off_t) (2 * span)) != 0) { + FAIL("file setup failed"); + if (fd2 >= 0) + close(fd2); + close(fd1); + return; + } + + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd1, 0); + void *base = first; + if (first == MAP_FAILED) { + FAIL("address reservation or first file mmap failed"); + close(fd2); + close(fd1); + return; + } + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd2, (off_t) span); + if (first != base || second != (char *) base + span) { + FAIL("adjacent file mmap failed"); + if (first == base) + munmap(first, span); + if (second == (char *) base + span) + munmap(second, span); + close(fd2); + close(fd1); + return; + } + + errno = 0; + void *q = mremap(base, 2 * span, 2 * span, 0); + if (q == MAP_FAILED && errno == EFAULT) + PASS(); + else + FAIL("mremap accepted two independent tracker records"); + + munmap(first, span); + munmap(second, span); + close(fd2); + close(fd1); +} + +static void test_file_backed_fork_split_move(void) +{ + TEST("MAP_SHARED file: mremap across fork-split source"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-cf-mremap-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0 || pwrite(fd, "F", 1, 0) != 1 || + pwrite(fd, "X", 1, (off_t) span) != 1) { + FAIL("file setup"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap"); + close(fd); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, span); + close(fd); + return; + } + + if (pid == 0) { + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(40); + if (grown[0] != 'F' || grown[span] != 0) + _exit(41); + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(42); + + /* A second fork marks both tracker records inherited. Their stable + * logical-VMA lineage must still allow the grandchild to mremap the + * complete source. + */ + pid_t grandchild = fork(); + if (grandchild < 0) + _exit(43); + if (grandchild == 0) { + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved == MAP_FAILED) + _exit(44); + if (moved == grown) + _exit(45); + if (moved[0] != 'F' || moved[span] != 0 || moved[2 * span] != 0) + _exit(46); + + munmap(moved, 3 * span); + munmap(blocker, span); + _exit(0); + } + + int grandchild_status = 0; + if (waitpid(grandchild, &grandchild_status, 0) < 0 || + !WIFEXITED(grandchild_status)) + _exit(47); + if (WEXITSTATUS(grandchild_status) != 0) + _exit(WEXITSTATUS(grandchild_status)); + munmap(grown, 2 * span); + munmap(blocker, span); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid"); + } else if (!WIFEXITED(status)) { + FAIL("child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "child mremap failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_file_backed_mprotect_fragments_move(void) +{ + TEST("MAP_SHARED file: mremap across restored mprotect fragments"); + + const size_t span = 64 * 1024; + const size_t old_size = 3 * span; + const size_t new_size = 4 * span; + char tmpl[] = "/tmp/elfuse-mremap-fragments-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) new_size) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(5 * span, old_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'A'; + p[span] = 'B'; + p[2 * span] = 'C'; + + if (mprotect(p + span, span, PROT_READ) != 0 || + mprotect(p + span, span, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect split and restore failed"); + munmap(p, old_size); + close(fd); + return; + } + + void *blocker = mmap(p + old_size, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != p + old_size) { + FAIL("move blocker mmap failed"); + munmap(p, old_size); + close(fd); + return; + } + + char *moved = mremap(p, old_size, new_size, MREMAP_MAYMOVE); + if (moved == MAP_FAILED || moved == p || moved[0] != 'A' || + moved[span] != 'B' || moved[2 * span] != 'C' || moved[3 * span] != 0) { + FAIL("fragmented logical VMA did not move intact"); + if (moved == MAP_FAILED) + munmap(p, old_size); + else + munmap(moved, new_size); + } else { + moved[3 * span] = 'D'; + PASS(); + munmap(moved, new_size); + } + + munmap(blocker, span); + close(fd); +} + +static void test_file_backed_fixed_move_from_same_vma(void) +{ + TEST("MAP_SHARED file: fixed subrange move keeps source fd live"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-fixed-source-fd-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (3 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'D'; + p[span] = 'M'; + p[2 * span] = 'S'; + + char *moved = + mremap(p + 2 * span, span, span, MREMAP_MAYMOVE | MREMAP_FIXED, p); + if (moved != p || moved[0] != 'S' || moved[span] != 'M') + FAIL("fixed subrange move lost its source backing fd"); + else + PASS(); + + munmap(p, 3 * span); + close(fd); +} + +static void test_file_backed_mremap_emfile_atomic(void) +{ + TEST("MAP_SHARED fork growth: mremap EMFILE preserves source"); + + const size_t span = 64 * 1024; + const size_t page_size = 4096; + const int fill_limit = 4096; + char tmpl[] = "/tmp/elfuse-mremap-emfile-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(p, span); + close(fd); + return; + } + if (pid == 0) { + const size_t filler_reservation_size = + (size_t) fill_limit * 2 * page_size; + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(60); + grown[span] = 'T'; + grown[2 * span - 1] = 'Z'; + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(61); + + char *fixed_probe = + reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, (off_t) span); + if (fixed_probe == MAP_FAILED) + _exit(72); + fixed_probe[0] = 'D'; + fixed_probe[2 * span] = 'S'; + + int last_dup = -1; + for (;;) { + int duplicated = dup(fd); + if (duplicated < 0) + break; + last_dup = duplicated; + } + if (errno != EMFILE || last_dup < 0) + _exit(62); + /* Keep one guest slot available for the /proc/self/maps proof below. + * The close also releases one host descriptor; the file-backed filler + * loop consumes it again before finding the actual host/table limit. */ + if (close(last_dup) != 0) + _exit(69); + + void *filler_base = mmap(NULL, filler_reservation_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (filler_base == MAP_FAILED || + munmap(filler_base, filler_reservation_size) != 0) + _exit(70); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < fill_limit; filler_count++) { + void *filler = mmap( + (char *) filler_base + (size_t) filler_count * 2 * page_size, + page_size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, fd, 0); + if (filler == MAP_FAILED) + break; + last_filler = filler; + } + if (filler_count == 0 || filler_count == fill_limit || errno != ENOMEM) + _exit(63); + + /* The first failed filler proves no host descriptor remains. Freeing + * one owned tracker fd lets mremap's prefix dup succeed and forces the + * second, tail-owned dup to hit EMFILE. + */ + if (munmap(last_filler, page_size) != 0) + _exit(64); + + /* With one host and one guest descriptor available, maps must be + * readable. Near the fixed tracker capacity, VMA count alone cannot + * prove whether tracker pressure or host descriptors ended the filler + * loop, so skip instead of attributing an ambiguous ENOMEM to the + * second mremap dup. */ + int maps_count = 0; + if (!count_maps_entries(&maps_count) || maps_count >= 4000) + _exit(77); + + errno = 0; + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved != MAP_FAILED || errno != ENOMEM) + _exit(65); + + volatile char *source = grown; + if (source[0] != 'I' || source[span] != 'T' || + source[2 * span - 1] != 'Z') + _exit(66); + if (mprotect(grown, 2 * span, PROT_READ) != 0 || source[0] != 'I' || + source[span] != 'T' || + mprotect(grown, 2 * span, PROT_READ | PROT_WRITE) != 0) + _exit(67); + source[1] = 'J'; + source[span + 1] = 'U'; + if (source[1] != 'J' || source[span + 1] != 'U') + _exit(68); + + /* Exactly one host descriptor is free again. A fixed subrange move + * consumes it for target tracking, then must fail atomically when the + * source-boundary snapshot cannot duplicate its backing fd. */ + errno = 0; + char *fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != MAP_FAILED || errno != ENOMEM || fixed_probe[0] != 'D' || + fixed_probe[2 * span] != 'S') + _exit(73); + + /* Release guest dup slots and filler tracker descriptors, then retry. + * A failed split that published backing_fd=-1 would make this retry + * fail even though descriptor capacity is now available. */ + for (int guest_fd = 0; guest_fd < 4096; guest_fd++) { + if (guest_fd != fd) + (void) close(guest_fd); + } + if (munmap(filler_base, filler_reservation_size) != 0) + _exit(74); + + fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != fixed_probe || fixed[0] != 'S') + _exit(75); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("EMFILE child terminated abnormally"); + } else if (WEXITSTATUS(status) == 77) { + printf("SKIP: fd exhaustion and region-table pressure are ambiguous\n"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "EMFILE child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_heap_tail_mprotect_then_grow(void) +{ + TEST("brk growth does not reuse protected heap tail"); + + const uintptr_t page_size = 4096; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(50); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(51); + if (!set_program_break(final_end)) + _exit(52); + + if (!mapping_is_rw_private(protected_end)) + _exit(54); + + if (!set_program_break(protected_end)) + _exit(55); + if (!set_program_break(final_end)) + _exit(56); + if (!mapping_is_rw_private(protected_end)) + _exit(57); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("new brk page inherited stale tail protection"); + + (void) set_program_break(original); +} + +static void test_heap_growth_with_full_region_table(void) +{ + TEST("brk growth with full region table has no overlapping heap headers"); + + const uintptr_t page_size = 4096; + const int fill_limit = 8192; + const int minimum_fill = 2048; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(80); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(81); + + int filled = 0; + for (; filled < fill_limit; filled++) { + int prot = (filled & 1) ? PROT_NONE : PROT_READ; + void *q = + mmap(NULL, page_size, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) + break; + } + if (filled < minimum_fill || filled == fill_limit || errno != ENOMEM) + _exit(82); + + if (!set_program_break(final_end)) + _exit(83); + volatile char *new_page = (char *) protected_end; + new_page[0] = 'H'; + if (new_page[0] != 'H') + _exit(84); + + int heap_count = 0; + if (!heap_headers_do_not_overlap(&heap_count) || heap_count < 2) + _exit(85); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("region-table child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[96]; + snprintf(buf, sizeof(buf), "region-table child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + (void) set_program_break(original); +} + +int main(void) +{ + printf("test-mremap-fork-tracking: elfuse mremap tracker tests\n"); + + test_postfork_adjacent_anon_rejected(); + test_adjacent_file_vmas_rejected(); + test_file_backed_fork_split_move(); + test_file_backed_mprotect_fragments_move(); + test_file_backed_fixed_move_from_same_vma(); + test_file_backed_mremap_emfile_atomic(); + test_heap_tail_mprotect_then_grow(); + test_heap_growth_with_full_region_table(); + + SUMMARY("test-mremap-fork-tracking"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-mremap-tail-emfile.c b/tests/test-mremap-tail-emfile.c new file mode 100644 index 00000000..99ce2466 --- /dev/null +++ b/tests/test-mremap-tail-emfile.c @@ -0,0 +1,235 @@ +/* + * test-mremap-tail-emfile exercises file-backed mremap bookkeeping while the + * guest and host descriptor tables are exhausted. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This probe deliberately uses libc interfaces so it can serve as the + * repository's portable C implementation of the regression. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif + +static const size_t PAGE_SIZE = 4096; +static const size_t SPAN = 64 * 1024; +static const int FILL_LIMIT = 4096; + +static void child_exit(int status) +{ + _exit(status); +} + +static void child_fail(int status) +{ + child_exit(status); +} + +static void child_probe(int file_fd, unsigned char *base) +{ + void *result = mremap(base, SPAN, 3 * SPAN, 0); + if (result == MAP_FAILED || result != base) + child_fail(10); + + base[SPAN] = 'T'; + base[2 * SPAN] = 'R'; + base[SPAN + PAGE_SIZE] = 'S'; + base[SPAN + 3 * PAGE_SIZE] = 'M'; + + /* Exhaust both descriptor tables, then leave one slot available. */ + int last_dup = -1; + int dup_error = 0; + for (;;) { + int duplicated = dup(file_fd); + if (duplicated < 0) { + dup_error = errno; + break; + } + last_dup = duplicated; + } + if (dup_error != EMFILE || last_dup < 0) + child_fail(11); + if (close(last_dup) != 0) + child_fail(12); + + /* Fill the region table with one-page file mappings separated by holes. */ + const size_t filler_length = (size_t) FILL_LIMIT * 2 * PAGE_SIZE; + void *reserved = mmap(NULL, filler_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) + child_fail(13); + if (munmap(reserved, filler_length) != 0) + child_fail(13); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < FILL_LIMIT; filler_count++) { + void *address = (char *) reserved + + (size_t) filler_count * 2 * PAGE_SIZE; + void *filler = mmap(address, PAGE_SIZE, PROT_NONE, + MAP_PRIVATE | MAP_FIXED, file_fd, 0); + if (filler == MAP_FAILED) { + int filler_errno = errno; + if (filler_errno != ENOMEM || filler_count == 0) + child_fail(filler_errno == ENOMEM ? 16 : 14); + break; + } + if (filler != address) + child_fail(15); + last_filler = filler; + } + if (filler_count == FILL_LIMIT || last_filler == MAP_FAILED) + child_fail(16); + if (munmap(last_filler, PAGE_SIZE) != 0) + child_fail(17); + + /* Consume the one descriptor released by the last filler mapping. */ + int probe_fd = dup(file_fd); + if (probe_fd < 0) + child_fail(22); + + /* Splitting either boundary of the child-private tail must fail before + * changing memory or PTEs when no descriptor is available. */ + errno = 0; + result = mremap(base, SPAN + 2 * PAGE_SIZE, SPAN + PAGE_SIZE, 0); + int remap_errno = errno; + if (result != MAP_FAILED || remap_errno != ENOMEM || + base[SPAN + PAGE_SIZE] != 'S') + child_fail(23); + + errno = 0; + int munmap_result = munmap(base + SPAN + 3 * PAGE_SIZE, PAGE_SIZE); + int munmap_errno = errno; + if (munmap_result != -1 || munmap_errno != ENOMEM || + base[SPAN + 3 * PAGE_SIZE] != 'M') + child_fail(25); + + if (close(probe_fd) != 0) + child_fail(27); + + /* old_size ends inside the child-private tail. */ + errno = 0; + result = mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, + MREMAP_MAYMOVE); + remap_errno = errno; + + /* Drop all duplicate guest slots, retaining the original file fd. */ + for (int fd = 0; fd < FILL_LIMIT; fd++) { + if (fd != file_fd) + (void) close(fd); + } + if (munmap(reserved, filler_length) != 0) + child_fail(18); + + /* A successful move owns a target mapping; release it before msync. */ + if (result != MAP_FAILED) { + if (munmap(result, 2 * SPAN + PAGE_SIZE) != 0) + child_fail(21); + } else if (remap_errno != ENOMEM) { + child_fail(21); + } + + if (msync(base + 2 * SPAN, SPAN, MS_SYNC) != 0) + child_fail(19); + + unsigned char byte = 0; + if (pread(file_fd, &byte, 1, 2 * SPAN) != 1 || byte != 'R') + child_fail(20); + + child_exit(0); +} + +static void test_file_backed_mremap_tail_emfile(void) +{ + TEST("MAP_SHARED mremap tail EMFILE preserves source"); + + char path[] = "/tmp/elfuse-mremap-tail-emfile-XXXXXX"; + int file_fd = mkstemp(path); + if (file_fd < 0) { + FAIL("create temp file failed"); + return; + } + (void) unlink(path); + + const size_t file_length = 4 * SPAN; + if (ftruncate(file_fd, (off_t) file_length) != 0) { + FAIL("truncate temp file failed"); + close(file_fd); + return; + } + + const size_t reservation_length = file_length + PAGE_SIZE; + void *reserved = mmap(NULL, reservation_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED || munmap(reserved, reservation_length) != 0) { + FAIL("reserve source range failed"); + close(file_fd); + return; + } + + void *source_address = (char *) reserved + PAGE_SIZE; + void *mapped = mmap(source_address, SPAN, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file_fd, 0); + if (mapped == MAP_FAILED || mapped != source_address) { + FAIL("map source file failed"); + close(file_fd); + return; + } + + unsigned char *base = mapped; + base[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) munmap(base, SPAN); + close(file_fd); + return; + } + if (pid == 0) + child_probe(file_fd, base); + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("wait failed"); + } else if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + char message[96]; + if (WIFEXITED(status)) + snprintf(message, sizeof(message), "child failed at step %d", + WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) + snprintf(message, sizeof(message), "child killed by signal %d", + WTERMSIG(status)); + else + snprintf(message, sizeof(message), "child ended unexpectedly"); + FAIL(message); + } else { + PASS(); + } + + (void) munmap(base, SPAN); + close(file_fd); +} + +int main(void) +{ + printf("test-mremap-tail-emfile: file-backed mremap atomicity\n"); + test_file_backed_mremap_tail_emfile(); + SUMMARY("test-mremap-tail-emfile"); + return fails != 0; +} diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 3563b22c..5be4802d 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -19,7 +19,6 @@ #include #include "test-harness.h" -#include "test-util.h" int passes = 0, fails = 0; @@ -367,11 +366,16 @@ static void free_smaps(smaps_info_t *info) * on both success and failure. */ static bool load_smaps(const char *path, smaps_info_t *info) { + FILE *file = fopen(path, "r"); + if (!file) + return false; + char *buf = NULL; - size_t len = 0; - bool ok = read_file_dynamic_nul(path, &buf, &len) >= 0 && - parse_smaps(buf, len, info); + size_t capacity = 0; + ssize_t length = getdelim(&buf, &capacity, '\0', file); + bool ok = length >= 0 && parse_smaps(buf, (size_t) length, info); free(buf); + (void) fclose(file); return ok; } @@ -462,13 +466,69 @@ static bool read_exact(int fd, void *data, size_t len) return true; } +static bool write_exact(int fd, const void *data, size_t len) +{ + const char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = write(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + static int child_probe(uintptr_t target, uintptr_t stress, size_t page_size, size_t stress_size) { - void *postfork = mmap(NULL, page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + void *postfork = MAP_FAILED; + uintptr_t target_end = target + 3 * page_size; + uintptr_t stress_end = stress + stress_size; + +#ifdef MAP_FIXED_NOREPLACE + /* Keep the probe in a known gap so the synthetic smaps builder cannot + * merge it with the target's final rw page or the stress fixture. */ + uintptr_t fixed_hint = 0x700000000000ULL; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) fixed_hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (candidate != MAP_FAILED) { + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + } + fixed_hint += 16 * page_size; + } +#endif + + if (postfork == MAP_FAILED) { + uintptr_t hint = stress_end + 64 * page_size; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (candidate == MAP_FAILED) + break; + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + hint += 64 * page_size; + } + } if (postfork == MAP_FAILED) return 1; @@ -590,7 +650,7 @@ int main(void) close(pipefd[0]); int result = child_probe((uintptr_t) target, (uintptr_t) stress, page_size, stress_size); - (void) write_fd_all(pipefd[1], &result, sizeof(result)); + (void) write_exact(pipefd[1], &result, sizeof(result)); close(pipefd[1]); _exit(result); } else { From fc4a3b920ed12dd620bb6c993a5c8f7cba68cc03 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 08:27:04 +0800 Subject: [PATCH 11/14] Format memory remap code Apply consistent line wrapping in the memory syscall implementation and its mremap regression test without changing behavior. --- src/syscall/mem.c | 9 +++------ tests/test-mremap-tail-emfile.c | 8 ++++---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 45943aa5..7ea6b2ef 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -357,8 +357,7 @@ static int find_mremap_source(const guest_t *g, if (!collect_segments) return 0; - source->segments = - malloc((size_t) nsegments * sizeof(*source->segments)); + source->segments = malloc((size_t) nsegments * sizeof(*source->segments)); if (!source->segments) { dispose_mremap_source(source); return -LINUX_ENOMEM; @@ -371,8 +370,7 @@ static int find_mremap_source(const guest_t *g, const guest_region_t *r = &g->regions[index + segment_index]; uint64_t segment_end = r->end < end ? r->end : end; uint64_t segment_len = segment_end - cursor; - mremap_source_segment_t *segment = - &source->segments[segment_index]; + mremap_source_segment_t *segment = &source->segments[segment_index]; segment->start = cursor; segment->end = segment_end; segment->gpa_base = expected_gpa; @@ -4108,8 +4106,7 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) close(remove_fd); return -LINUX_ENOMEM; } - guest_region_remove_reserved(g, addr, addr + length, - remove_fd); + guest_region_remove_reserved(g, addr, addr + length, remove_fd); } return 0; } diff --git a/tests/test-mremap-tail-emfile.c b/tests/test-mremap-tail-emfile.c index 99ce2466..ce94e872 100644 --- a/tests/test-mremap-tail-emfile.c +++ b/tests/test-mremap-tail-emfile.c @@ -79,8 +79,8 @@ static void child_probe(int file_fd, unsigned char *base) void *last_filler = MAP_FAILED; int filler_count = 0; for (; filler_count < FILL_LIMIT; filler_count++) { - void *address = (char *) reserved + - (size_t) filler_count * 2 * PAGE_SIZE; + void *address = + (char *) reserved + (size_t) filler_count * 2 * PAGE_SIZE; void *filler = mmap(address, PAGE_SIZE, PROT_NONE, MAP_PRIVATE | MAP_FIXED, file_fd, 0); if (filler == MAP_FAILED) { @@ -124,8 +124,8 @@ static void child_probe(int file_fd, unsigned char *base) /* old_size ends inside the child-private tail. */ errno = 0; - result = mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, - MREMAP_MAYMOVE); + result = + mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, MREMAP_MAYMOVE); remap_errno = errno; /* Drop all duplicate guest slots, retaining the original file fd. */ From c4efc5e23ff531adc2c77794703c5d95137cadc3 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 15:29:47 +0800 Subject: [PATCH 12/14] Harden fork-aware smaps and mremap Preserve fork lineage and smaps consistency when restored regions receive new allocations. Flush shared-file contents before mremap removes source mappings. Make region-boundary preparation transactional so allocation or descriptor failures leave metadata unchanged. Add regressions for repeated post-fork allocations and misaligned moves. Cover PROT_NONE smaps output as well. Related: #259 --- docs/internals.md | 27 +++-- src/core/guest.c | 10 ++ src/core/guest.h | 5 + src/runtime/fork-state.c | 1 + src/runtime/procemu.c | 27 +++-- src/syscall/mem.c | 189 +++++++++++++++++++++++++++++- tests/test-mremap-fork-tracking.c | 147 +++++++++++++++++++++++ tests/test-proc-smap.c | 82 ++++++++----- 8 files changed, 430 insertions(+), 58 deletions(-) diff --git a/docs/internals.md b/docs/internals.md index 3096dc26..32163846 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -796,28 +796,29 @@ under `/proc`, `/dev`, and a few Linux-expected compatibility files: `/proc/self/smaps` and `/proc//smaps` are generated from the same tracked VMA list as `/proc/self/maps`. The complete field set currently emitted for -each VMA is the maps header followed by these 25 fields, in this order: +each VMA is the maps header followed by these 24 fields, in this order: ```text Size, KernelPageSize, MMUPageSize, Rss, Pss, Pss_Dirty, Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty, Referenced, Anonymous, KSM, LazyFree, AnonHugePages, ShmemPmdMapped, FilePmdMapped, Shared_Hugetlb, Private_Hugetlb, Swap, SwapPss, Locked, THPeligible, -ProtectionKey, VmFlags +VmFlags ``` `Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are -reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a -non-zero value: in a fork child, writable private anonymous VMAs that existed -in the parent's CoW snapshot report their full VMA size because they are -logically shared with that snapshot. Every other numeric counter, including -`THPeligible` and `ProtectionKey`, is emitted as zero. `VmFlags` is -evidence-based and contains only the permission/sharing flags represented by -the tracked VMA (`rd`, `wr`, -`ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. -elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing -against a real Linux kernel should tolerate either conditional field being -absent. +reported as 4 KiB. In a fork child, writable private anonymous VMAs that +existed in the parent's CoW snapshot report their full VMA size for +`Shared_Dirty`, `Rss`, and `Pss`, keeping those coarse counters internally +consistent. Newly-created VMAs are excluded from that signal. Every other +numeric counter, including `THPeligible`, is emitted as a stable zero. +`ProtectionKey` (a Linux pkeys field added in Linux 4.9) is intentionally +omitted because the macOS host has no equivalent; consumers comparing against a +real Linux kernel should treat it as optional. `VmFlags` is evidence-based and +contains only the permission/sharing flags represented by the tracked VMA +(`rd`, `wr`, `ex`, `sh`, and `nr` when applicable); no untracked kernel flags +are invented. A VMA with no such evidence (for example `PROT_NONE`) still +prints the field's separating space as `VmFlags: `. These are coarse VMA-level values, not host page-residency, dirty-bit, or proportional-sharing accounting, so they are suitable for fork-safety checks but not precise memory profiling. The fork-child marker is tracked per VMA, so diff --git a/src/core/guest.c b/src/core/guest.c index 7620dcb2..d3d2ebf2 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -2040,6 +2040,16 @@ static uint64_t allocate_vma_id(guest_t *g) } } +void guest_reseed_next_vma_id(guest_t *g) +{ + uint64_t max_id = g->next_vma_id; + for (int i = 0; i < g->nregions; i++) { + if (g->regions[i].vma_id > max_id) + max_id = g->regions[i].vma_id; + } + g->next_vma_id = max_id; +} + int guest_region_add_ex_owned(guest_t *g, uint64_t start, uint64_t end, diff --git a/src/core/guest.h b/src/core/guest.h index 0645e70b..c16dca56 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -1214,6 +1214,11 @@ int guest_region_add_ex_owned_gpa(guest_t *g, bool inherited_at_fork, uint64_t vma_id); +/* Re-seed the logical-VMA allocator from a restored region snapshot. Fork IPC + * restores regions by value, so next_vma_id must be advanced past every + * serialized lineage before child-private mappings are added. */ +void guest_reseed_next_vma_id(guest_t *g); + /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest * MAP_FIXED_NOREPLACE reservations. diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 9ff5a799..49f0c6ac 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -933,6 +933,7 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) */ for (int i = 0; i < g->nregions; i++) g->regions[i].inherited_at_fork = true; + guest_reseed_next_vma_id(g); g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 97911b31..1c8a015d 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -2649,8 +2649,10 @@ static int proc_open_self_maps(const guest_t *g) * guest VMAs and logical fork snapshots, but it cannot observe kernel page * residency or dirty bits on the host. Writable private anonymous VMAs that * were present in the most recent fork snapshot report their full VMA size as - * Shared_Dirty; newly-created VMAs are excluded. All other fields that - * require kernel page accounting are stable zeroes. + * Shared_Dirty, Rss, and Pss; keeping those counters aligned avoids a + * self-contradictory snapshot while retaining a coarse fork-compatibility + * signal. Newly-created VMAs are excluded. All other fields that require + * kernel page accounting are stable zeroes. */ static int proc_open_self_smaps(const guest_t *g) { @@ -2681,15 +2683,22 @@ static int proc_open_self_smaps(const guest_t *g) bool logical_shared_dirty = e->inherited_at_fork && private_anon && (e->prot & LINUX_PROT_WRITE); uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; + uint64_t rss_kb = shared_dirty_kb; + uint64_t pss_kb = shared_dirty_kb; - char vmflags[64]; - size_t vmflags_len = 0; + /* Linux writes a separating space after VmFlags:, even when no + * evidence-based flags are available (for example a PROT_NONE VMA). + * Start with that space so the empty form is exactly "VmFlags: \n". + */ + char vmflags[64] = {' '}; + size_t vmflags_len = 1; #define APPEND_VMFLAG(flag) \ do { \ const char *token = (flag); \ size_t token_len = strlen(token); \ if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ - vmflags[vmflags_len++] = ' '; \ + if (vmflags_len > 1) \ + vmflags[vmflags_len++] = ' '; \ memcpy(vmflags + vmflags_len, token, token_len); \ vmflags_len += token_len; \ } \ @@ -2717,8 +2726,8 @@ static int proc_open_self_smaps(const guest_t *g) "Size: %llu kB\n" "KernelPageSize: 4 kB\n" "MMUPageSize: 4 kB\n" - "Rss: 0 kB\n" - "Pss: 0 kB\n" + "Rss: %llu kB\n" + "Pss: %llu kB\n" "Pss_Dirty: 0 kB\n" "Shared_Clean: 0 kB\n" "Shared_Dirty: %llu kB\n" @@ -2737,9 +2746,9 @@ static int proc_open_self_smaps(const guest_t *g) "SwapPss: 0 kB\n" "Locked: 0 kB\n" "THPeligible: 0\n" - "ProtectionKey: 0\n" "VmFlags:%s\n", header_len, header, (unsigned long long) size_kb, + (unsigned long long) rss_kb, (unsigned long long) pss_kb, (unsigned long long) shared_dirty_kb, vmflags) < 0) goto out; } @@ -3360,7 +3369,7 @@ int proc_intercept_open(const guest_t *g, return proc_open_self_maps(g); /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata - * and the coarse fork Shared_Dirty compatibility signal. + * and the coarse fork Shared_Dirty/Rss/Pss compatibility signal. */ if (!strcmp(path, "/proc/self/smaps")) return proc_open_self_smaps(g); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 7ea6b2ef..2d18ed8b 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -105,6 +105,10 @@ static int restore_snapshot_page_tables(guest_t *g, static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n); +static int64_t sync_shared_aliases_range(guest_t *g, + int backing_fd, + uint64_t file_start, + uint64_t file_end); static int region_count_after_removes(const guest_t *g, const remove_range_t *ranges, @@ -251,6 +255,7 @@ typedef struct { uint64_t gpa_base; uint64_t offset; int backing_fd; /* borrowed from the region tracker */ + bool shared_non_overlay; bool overlay_active; uint64_t overlay_start; uint64_t overlay_end; @@ -376,6 +381,8 @@ static int find_mremap_source(const guest_t *g, segment->gpa_base = expected_gpa; segment->offset = expected_offset; segment->backing_fd = r->backing_fd; + segment->shared_non_overlay = + r->shared && r->backing_fd >= 0 && !region_has_live_overlay(r); segment->overlay_active = region_has_live_overlay(r); segment->overlay_start = r->overlay_start; segment->overlay_end = r->overlay_end; @@ -460,6 +467,71 @@ static void region_clip_overlay(guest_region_t *r) region_clear_overlay(r); } +/* Region-boundary edits are a prepare step for metadata snapshots. Keep a + * private copy of the tracker while the boundaries are split so an allocation + * failure (table capacity or backing-fd dup) cannot publish a half-split + * tracker. Split-created fds are closed on rollback; the original region fds + * remain owned by the original array and are restored verbatim. + */ +typedef struct { + guest_region_t *regions; + int nregions; +} region_array_txn_t; + +static int begin_region_array_txn(const guest_t *g, region_array_txn_t *txn) +{ + memset(txn, 0, sizeof(*txn)); + txn->nregions = g->nregions; + if (txn->nregions == 0) + return 0; + + txn->regions = malloc((size_t) txn->nregions * sizeof(*txn->regions)); + if (!txn->regions) { + txn->nregions = 0; + return -LINUX_ENOMEM; + } + memcpy(txn->regions, g->regions, + (size_t) txn->nregions * sizeof(*txn->regions)); + return 0; +} + +static bool txn_original_fd_present(const region_array_txn_t *txn, int fd) +{ + if (fd < 0) + return true; + for (int i = 0; i < txn->nregions; i++) { + if (txn->regions[i].backing_fd == fd) + return true; + } + return false; +} + +static void rollback_region_array_txn(guest_t *g, region_array_txn_t *txn) +{ + if (!txn || !txn->regions) + return; + + /* split_regions_at_boundary() only creates new fds; it never closes an + * original one. Close those new descriptors before restoring the copy. + */ + for (int i = 0; i < g->nregions; i++) { + int fd = g->regions[i].backing_fd; + if (fd >= 0 && !txn_original_fd_present(txn, fd)) + close(fd); + } + memcpy(g->regions, txn->regions, + (size_t) txn->nregions * sizeof(*txn->regions)); + g->nregions = txn->nregions; +} + +static void finish_region_array_txn(region_array_txn_t *txn) +{ + if (!txn) + return; + free(txn->regions); + memset(txn, 0, sizeof(*txn)); +} + static int split_regions_at_boundary(guest_t *g, uint64_t boundary) { if (boundary == 0) @@ -1288,6 +1360,34 @@ static bool mremap_source_has_overlay(const mremap_source_t *source) return false; } +/* A snapshot-style MAP_SHARED source may contain guest writes that have not + * reached its backing file yet. mremap destroys the source VMA (and may zero + * its slab backing) after copying, so publish those dirty bytes before any + * source cleanup. Live file overlays are excluded: the host page cache already + * owns their coherence and cleanup restores the slab before the source is + * removed. + */ +static int64_t flush_mremap_source_shared(guest_t *g, + const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + if (!segment->shared_non_overlay || segment->backing_fd < 0) + continue; + + uint64_t len = segment->end - segment->start; + uint64_t file_end = segment->offset + len; + if (file_end < segment->offset) + return -LINUX_EFAULT; + + int64_t err = sync_shared_aliases_range(g, segment->backing_fd, + segment->offset, file_end); + if (err < 0) + return err; + } + return 0; +} + typedef struct { uint64_t start; uint64_t end; @@ -1487,12 +1587,26 @@ static int capture_region_snapshots(guest_t *g, region_snapshot_t *snaps, int max_snaps) { + /* Split and snapshot as one metadata transaction. A failed boundary split, + * descriptor dup, or snapshot-capacity check must leave regions[] and its + * owned fds exactly as they were on entry. */ + region_array_txn_t txn; + int txn_err = begin_region_array_txn(g, &txn); + if (txn_err < 0) + return txn_err; + int split_err = split_regions_at_boundary(g, start); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return split_err; + } split_err = split_regions_at_boundary(g, end); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return split_err; + } int n = 0; for (int i = 0; i < g->nregions; i++) { @@ -1503,6 +1617,8 @@ static int capture_region_snapshots(guest_t *g, continue; if (n >= max_snaps) { close_region_snapshots(snaps, n); + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return -LINUX_ENOMEM; } @@ -1519,6 +1635,8 @@ static int capture_region_snapshots(guest_t *g, snap->backing_fd = dup(r->backing_fd); if (snap->backing_fd < 0) { close_region_snapshots(snaps, n); + rollback_region_array_txn(g, &txn); + finish_region_array_txn(&txn); return -LINUX_ENOMEM; } } @@ -1530,6 +1648,7 @@ static int capture_region_snapshots(guest_t *g, str_copy_trunc(snap->name, r->name, sizeof(snap->name)); } + finish_region_array_txn(&txn); return n; } @@ -2122,12 +2241,23 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) uint64_t host_start = ALIGN_DOWN(start, hps); uint64_t host_end = ALIGN_UP(end, hps); + region_array_txn_t split_txn; + int txn_err = begin_region_array_txn(g, &split_txn); + if (txn_err < 0) + return txn_err; int split_err = split_regions_at_boundary(g, host_start); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &split_txn); + finish_region_array_txn(&split_txn); return split_err; + } split_err = split_regions_at_boundary(g, host_end); - if (split_err < 0) + if (split_err < 0) { + rollback_region_array_txn(g, &split_txn); + finish_region_array_txn(&split_txn); return split_err; + } + finish_region_array_txn(&split_txn); /* Snapshot affected ranges first; the host-side mmap calls below do not * touch the region array, but a future caller invariant is to allow this @@ -3313,9 +3443,27 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, -LINUX_ENOMEM); } + /* Keep both boundary captures in one transaction. The per-capture + * helper rolls back its own edits, while this outer guard also undoes + * a successful source capture if destination capture or the preflight + * shared-file flush fails afterward. */ + region_array_txn_t capture_txn; + int capture_txn_err = begin_region_array_txn(g, &capture_txn); + if (capture_txn_err < 0) { + free(source_snaps); + free(dest_snaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, capture_txn_err); + } + source_nsnaps = capture_region_snapshots( g, old_off, old_off + old_size, source_snaps, GUEST_MAX_REGIONS); if (source_nsnaps < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); free(source_snaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3327,6 +3475,8 @@ int64_t sys_mremap(guest_t *g, int rebind_err = rebind_mremap_source_backings(&source, source_snaps, source_nsnaps); if (rebind_err < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); dispose_region_snapshots(&source_snaps, &source_nsnaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3338,6 +3488,8 @@ int64_t sys_mremap(guest_t *g, dest_nsnaps = capture_region_snapshots(g, new_off, new_off + new_size, dest_snaps, GUEST_MAX_REGIONS); if (dest_nsnaps < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); dispose_region_snapshots(&source_snaps, &source_nsnaps); free(dest_snaps); if (track_backing_fd >= 0) @@ -3347,6 +3499,20 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, dest_nsnaps); } + int64_t flush_err = flush_mremap_source_shared(g, &source); + if (flush_err < 0) { + rollback_region_array_txn(g, &capture_txn); + finish_region_array_txn(&capture_txn); + dispose_region_snapshots(&dest_snaps, &dest_nsnaps); + dispose_region_snapshots(&source_snaps, &source_nsnaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, flush_err); + } + finish_region_array_txn(&capture_txn); + if (source_overlay) { int cleanup_err = cleanup_overlays_in_range(g, old_off, old_off + old_size); @@ -3681,6 +3847,15 @@ int64_t sys_mremap(guest_t *g, return finish_mremap(&source, -LINUX_ENOMEM); } + int64_t flush_err = flush_mremap_source_shared(g, &source); + if (flush_err < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, flush_err); + } + int source_remove_fd = -1; if (guest_region_remove_prepare(g, old_off, old_off + old_size, &source_remove_fd) < 0) { @@ -4312,8 +4487,8 @@ static int64_t sync_shared_aliases_range(guest_t *g, uint8_t original[4096]; for (uint64_t chunk_start = file_start; chunk_start < file_end;) { - uint64_t chunk_end = ALIGN_DOWN(chunk_start + sizeof(original), 4096); - if (chunk_end <= chunk_start || chunk_end > file_end) + uint64_t chunk_end = chunk_start + sizeof(original); + if (chunk_end < chunk_start || chunk_end > file_end) chunk_end = file_end; size_t chunk_len = (size_t) (chunk_end - chunk_start); @@ -4329,6 +4504,8 @@ static int64_t sync_shared_aliases_range(guest_t *g, const guest_region_t *src = &g->regions[i]; if (!src->shared || src->backing_fd < 0) continue; + if (src->overlay_active) + continue; if (!(src->prot & LINUX_PROT_WRITE)) continue; if (!same_backing_file(backing_fd, src->backing_fd)) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index 0bdf9b31..bac30b2e 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -261,6 +261,55 @@ static void test_postfork_adjacent_anon_rejected(void) munmap(first, span); } +static void test_postfork_repeated_allocations_keep_lineage(void) +{ + TEST("repeated post-fork allocations never alias inherited VMA IDs"); + + const size_t span = 64 * 1024; + const int attempts = 256; + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (first == MAP_FAILED) { + FAIL("inherited anonymous mmap failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(first, span); + return; + } + if (pid == 0) { + for (int i = 0; i < attempts; i++) { + void *tail = + mmap((char *) first + span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (tail != (char *) first + span) + _exit(90); + + /* If child allocation reused the inherited source's vma_id, + * find_mremap_source would incorrectly accept the adjacent pair. + * A reseeded allocator keeps the logical lineages distinct. */ + errno = 0; + void *same = mremap(first, 2 * span, 2 * span, 0); + if (same != MAP_FAILED || errno != EFAULT) + _exit(91); + if (munmap(tail, span) != 0) + _exit(92); + } + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("post-fork VMA IDs collided during repeated allocation"); + munmap(first, span); +} + static void test_adjacent_file_vmas_rejected(void) { TEST("mremap rejects adjacent independent file VMAs"); @@ -317,6 +366,102 @@ static void test_adjacent_file_vmas_rejected(void) close(fd1); } +static void test_misaligned_shared_mremap_writeback(void) +{ + TEST("misaligned MAP_SHARED mremap writes back after move"); + + const size_t page = 4096; + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-misaligned-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (2 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + /* Deliberately place the source one host page into its reservation. This + * keeps the guest mapping page-aligned while making it non-2MiB-aligned, + * so the MAP_SHARED overlay/mremap path exercises a split HVF segment. */ + size_t reservation_length = 2 * span + page; + void *reservation = mmap(NULL, reservation_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED || + munmap(reservation, reservation_length) != 0) { + FAIL("source reservation failed"); + if (reservation != MAP_FAILED) + (void) munmap(reservation, reservation_length); + close(fd); + return; + } + char *source_address = (char *) reservation + page; + char *source = mmap(source_address, span, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd, 0); + if (source == MAP_FAILED || source != source_address) { + FAIL("misaligned MAP_SHARED mmap failed"); + close(fd); + return; + } + source[0] = 'A'; + source[page + 7] = 'B'; + + /* Block in-place growth so mremap must move the mapping and leave the + * destination on the snapshot-style shared-writeback path. */ + void *blocker = mmap(source + span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != source + span) { + FAIL("mremap blocker mmap failed"); + munmap(source, span); + close(fd); + return; + } + + char *moved = mremap(source, span, 2 * span, MREMAP_MAYMOVE); + if (moved == MAP_FAILED || moved == source) { + FAIL("misaligned MAP_SHARED mremap did not move"); + if (moved != MAP_FAILED) + munmap(moved, 2 * span); + else + munmap(source, span); + munmap(blocker, span); + close(fd); + return; + } + if (moved[0] != 'A' || moved[page + 7] != 'B') { + FAIL("mremap move corrupted source bytes"); + munmap(moved, 2 * span); + munmap(blocker, span); + close(fd); + return; + } + + moved[page + 7] = 'C'; + moved[span + 19] = 'D'; + errno = 0; + int sync_result = msync(moved, 2 * span, MS_SYNC); + int sync_errno = errno; + unsigned char first = 0, extension = 0; + bool writeback_ok = sync_result == 0 && + pread(fd, &first, 1, (off_t) (page + 7)) == 1 && + pread(fd, &extension, 1, (off_t) (span + 19)) == 1 && + first == 'C' && extension == 'D'; + if (writeback_ok) + PASS(); + else { + errno = sync_errno; + FAIL("mremap destination did not write back MAP_SHARED bytes"); + } + + munmap(moved, 2 * span); + munmap(blocker, span); + close(fd); +} + static void test_file_backed_fork_split_move(void) { TEST("MAP_SHARED file: mremap across fork-split source"); @@ -829,7 +974,9 @@ int main(void) printf("test-mremap-fork-tracking: elfuse mremap tracker tests\n"); test_postfork_adjacent_anon_rejected(); + test_postfork_repeated_allocations_keep_lineage(); test_adjacent_file_vmas_rejected(); + test_misaligned_shared_mremap_writeback(); test_file_backed_fork_split_move(); test_file_backed_mprotect_fragments_move(); test_file_backed_fixed_move_from_same_vma(); diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 5be4802d..0096c443 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -30,8 +30,12 @@ typedef struct { unsigned long long size_kb; unsigned long long kernel_page_kb; unsigned long long mmu_page_kb; + unsigned long long rss_kb; + unsigned long long pss_kb; unsigned long long shared_dirty_kb; bool have_size; + bool have_rss; + bool have_pss; bool have_shared_dirty; bool have_vmflags; bool vmflags_wr; @@ -43,8 +47,7 @@ static const char *const smaps_fields[] = { "Private_Clean:", "Private_Dirty:", "Referenced:", "Anonymous:", "KSM:", "LazyFree:", "AnonHugePages:", "ShmemPmdMapped:", "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", "Swap:", - "SwapPss:", "Locked:", "THPeligible:", "ProtectionKey:", - "VmFlags:", + "SwapPss:", "Locked:", "THPeligible:", "VmFlags:", }; #define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) @@ -165,8 +168,12 @@ static bool parse_header(const char *line, smaps_vma_t *vma) vma->size_kb = 0; vma->kernel_page_kb = 0; vma->mmu_page_kb = 0; + vma->rss_kb = 0; + vma->pss_kb = 0; vma->shared_dirty_kb = 0; vma->have_size = false; + vma->have_rss = false; + vma->have_pss = false; vma->have_shared_dirty = false; vma->have_vmflags = false; vma->vmflags_wr = false; @@ -211,6 +218,12 @@ static bool parse_vmflags(const char *line, bool *writable) return false; const char *p = line + label_len; + if (*p && !isspace((unsigned char) *p)) + return false; + /* Keep the provider's stable spelling for an empty flag set: the Linux + * field always has a separating space, even when no token follows. */ + if (!*p) + return false; bool has_wr = false; while (*(p = skip_space(p))) { const char *start = p; @@ -233,21 +246,8 @@ static bool parse_vmflags(const char *line, bool *writable) static bool finish_vma(smaps_vma_t *vma, size_t field_index) { return field_index == SMAPS_FIELD_COUNT && vma->have_size && - vma->have_shared_dirty && vma->have_vmflags; -} - -/* THPeligible and ProtectionKey are conditional in real Linux kernels. The - * elfuse provider always emits both, but qemu may legitimately omit either; - * when the parser reaches one of those labels, advance over any missing - * optional fields before parsing VmFlags. */ -static void skip_optional_fields(const char *line, size_t *field_index) -{ - if (*field_index == SMAPS_FIELD_COUNT - 3 && - (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || - !strncmp(line, "VmFlags:", 8))) - (*field_index)++; - if (*field_index == SMAPS_FIELD_COUNT - 2 && !strncmp(line, "VmFlags:", 8)) - (*field_index)++; + vma->have_rss && vma->have_pss && vma->have_shared_dirty && + vma->have_vmflags; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) @@ -278,14 +278,10 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) goto fail; *next = '\0'; - /* Linux normally places headers back-to-back; the synthetic proc - * provider separates records with one blank line. Accept that - * separator only after a complete record. */ + /* smaps records are contiguous: a blank line is not a field and is + * rejected so malformed snapshots cannot hide an out-of-order VMA. */ if (!*line) { - if (!have_current || field_index != SMAPS_FIELD_COUNT) - goto fail; - line = next + 1; - continue; + goto fail; } smaps_vma_t header; @@ -306,7 +302,6 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) } else { if (!have_current) goto fail; - skip_optional_fields(line, &field_index); if (field_index >= SMAPS_FIELD_COUNT) goto fail; unsigned long long value; @@ -320,10 +315,18 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.kernel_page_kb = value; if (field_index == 2) current.mmu_page_kb = value; + if (field_index == 3) + current.rss_kb = value; + if (field_index == 4) + current.pss_kb = value; if (field_index == 7) current.shared_dirty_kb = value; if (field_index == 0) current.have_size = true; + if (field_index == 3) + current.have_rss = true; + if (field_index == 4) + current.have_pss = true; if (field_index == 7) current.have_shared_dirty = true; field_index++; @@ -432,10 +435,16 @@ static bool validate_layout(const smaps_info_t *info, middle->mmu_page_kb != 4 || last->kernel_page_kb != 4 || last->mmu_page_kb != 4) return false; + if (first->rss_kb != first->shared_dirty_kb || + first->pss_kb != first->shared_dirty_kb || middle->rss_kb != 0 || + middle->pss_kb != 0 || last->rss_kb != last->shared_dirty_kb || + last->pss_kb != last->shared_dirty_kb) + return false; - /* Every stress-map page alternates permissions, so each page must remain - * its own VMA. Requiring the full count makes a short read or a producer - * cap observable instead of merely checking that some blocks exceed 256. + /* Every stress-map page has deliberately distinct permissions (with one + * PROT_NONE probe), so each page must remain its own VMA. Requiring the + * full count makes a short read or a producer cap observable instead of + * merely checking that some blocks exceed 256. */ size_t expected_stress_vmas = stress_size / page_size; if (expected_stress_vmas <= 256 || @@ -448,6 +457,11 @@ static bool validate_layout(const smaps_info_t *info, page->end != stress + (i + 1) * page_size) return false; } + const smaps_vma_t *stress_none = find_vma(info, stress + 2 * page_size); + if (!stress_none || strcmp(stress_none->perms, "---p") || + stress_none->vmflags_wr || stress_none->shared_dirty_kb != 0 || + stress_none->rss_kb != 0 || stress_none->pss_kb != 0) + return false; return true; } @@ -549,13 +563,16 @@ static int child_probe(uintptr_t target, const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); const smaps_vma_t *stress_ro = find_vma(&info, stress); const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + const smaps_vma_t *stress_none = + find_vma(&info, stress + 2 * page_size); const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || - !postfork_vma || first->shared_dirty_kb == 0 || + !stress_none || !postfork_vma || first->shared_dirty_kb == 0 || middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || stress_ro->shared_dirty_kb != 0 || stress_rw->shared_dirty_kb == 0 || - postfork_vma->shared_dirty_kb != 0) { + stress_none->shared_dirty_kb != 0 || stress_none->rss_kb != 0 || + stress_none->pss_kb != 0 || postfork_vma->shared_dirty_kb != 0) { free_smaps(&info); munmap(postfork, page_size); return 1; @@ -604,6 +621,11 @@ int main(void) break; } } + /* Keep one page inaccessible so the parser and formatter exercise + * PROT_NONE coverage and the required empty VmFlags spelling. */ + if (fixture_ok && + mprotect(stress + 2 * page_size, page_size, PROT_NONE) < 0) + fixture_ok = false; } TEST("smaps headers, order, fields, and completeness"); From 03b28cad6225f90eb569ca810632f85312630402 Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 16:11:09 +0800 Subject: [PATCH 13/14] Align smaps dirty accounting and mremap flushes Keep synthesized Pss_Dirty consistent with fork-inherited Shared_Dirty, and avoid flushing writable aliases when moving a read-only MAP_SHARED snapshot. Add regressions for both cases. --- src/runtime/procemu.c | 10 ++-- src/syscall/mem.c | 17 ++++--- tests/test-mremap-fork-tracking.c | 82 +++++++++++++++++++++++++++++++ tests/test-proc-smap.c | 31 +++++++++--- 4 files changed, 122 insertions(+), 18 deletions(-) diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index 1c8a015d..ef37e6c3 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -2649,8 +2649,8 @@ static int proc_open_self_maps(const guest_t *g) * guest VMAs and logical fork snapshots, but it cannot observe kernel page * residency or dirty bits on the host. Writable private anonymous VMAs that * were present in the most recent fork snapshot report their full VMA size as - * Shared_Dirty, Rss, and Pss; keeping those counters aligned avoids a - * self-contradictory snapshot while retaining a coarse fork-compatibility + * Shared_Dirty, Rss, Pss, and Pss_Dirty; keeping those counters aligned avoids + * a self-contradictory snapshot while retaining a coarse fork-compatibility * signal. Newly-created VMAs are excluded. All other fields that require * kernel page accounting are stable zeroes. */ @@ -2685,6 +2685,7 @@ static int proc_open_self_smaps(const guest_t *g) uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; uint64_t rss_kb = shared_dirty_kb; uint64_t pss_kb = shared_dirty_kb; + uint64_t pss_dirty_kb = shared_dirty_kb; /* Linux writes a separating space after VmFlags:, even when no * evidence-based flags are available (for example a PROT_NONE VMA). @@ -2728,7 +2729,7 @@ static int proc_open_self_smaps(const guest_t *g) "MMUPageSize: 4 kB\n" "Rss: %llu kB\n" "Pss: %llu kB\n" - "Pss_Dirty: 0 kB\n" + "Pss_Dirty: %llu kB\n" "Shared_Clean: 0 kB\n" "Shared_Dirty: %llu kB\n" "Private_Clean: 0 kB\n" @@ -2749,6 +2750,7 @@ static int proc_open_self_smaps(const guest_t *g) "VmFlags:%s\n", header_len, header, (unsigned long long) size_kb, (unsigned long long) rss_kb, (unsigned long long) pss_kb, + (unsigned long long) pss_dirty_kb, (unsigned long long) shared_dirty_kb, vmflags) < 0) goto out; } @@ -3369,7 +3371,7 @@ int proc_intercept_open(const guest_t *g, return proc_open_self_maps(g); /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata - * and the coarse fork Shared_Dirty/Rss/Pss compatibility signal. + * and the coarse fork Shared_Dirty/Rss/Pss/Pss_Dirty compatibility signal. */ if (!strcmp(path, "/proc/self/smaps")) return proc_open_self_smaps(g); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index 2d18ed8b..df463efc 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -255,6 +255,7 @@ typedef struct { uint64_t gpa_base; uint64_t offset; int backing_fd; /* borrowed from the region tracker */ + /* Snapshot-style MAP_SHARED source that may contain guest writes. */ bool shared_non_overlay; bool overlay_active; uint64_t overlay_start; @@ -382,7 +383,8 @@ static int find_mremap_source(const guest_t *g, segment->offset = expected_offset; segment->backing_fd = r->backing_fd; segment->shared_non_overlay = - r->shared && r->backing_fd >= 0 && !region_has_live_overlay(r); + r->shared && (r->prot & LINUX_PROT_WRITE) && r->backing_fd >= 0 && + !region_has_live_overlay(r); segment->overlay_active = region_has_live_overlay(r); segment->overlay_start = r->overlay_start; segment->overlay_end = r->overlay_end; @@ -1360,12 +1362,13 @@ static bool mremap_source_has_overlay(const mremap_source_t *source) return false; } -/* A snapshot-style MAP_SHARED source may contain guest writes that have not - * reached its backing file yet. mremap destroys the source VMA (and may zero - * its slab backing) after copying, so publish those dirty bytes before any - * source cleanup. Live file overlays are excluded: the host page cache already - * owns their coherence and cleanup restores the slab before the source is - * removed. +/* A writable snapshot-style MAP_SHARED source may contain guest writes that + * have not reached its backing file yet. mremap destroys the source VMA (and + * may zero its slab backing) after copying, so publish those dirty bytes before + * any source cleanup. Read-only sources cannot contain writes and therefore do + * not trigger this alias scan. Live file overlays are excluded: the host page + * cache already owns their coherence and cleanup restores the slab before the + * source is removed. */ static int64_t flush_mremap_source_shared(guest_t *g, const mremap_source_t *source) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index bac30b2e..7649a180 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -462,6 +462,87 @@ static void test_misaligned_shared_mremap_writeback(void) close(fd); } +/* Force the snapshot-style MAP_SHARED path by placing a guest-page mapping + * one page into an otherwise unused reservation. Apple hosts use larger host + * pages than the guest's 4 KiB pages, so this address cannot receive a live + * file overlay. */ +static void *map_misaligned_shared_fixed(size_t length, + int prot, + int fd, + off_t offset) +{ + const size_t page = 4096; + void *reservation = mmap(NULL, length + page, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reservation == MAP_FAILED) + return MAP_FAILED; + + void *address = (char *) reservation + page; + if (munmap(reservation, length + page) != 0) + return MAP_FAILED; + return mmap(address, length, prot, MAP_SHARED | MAP_FIXED, fd, offset); +} + +static void test_readonly_shared_mremap_does_not_flush_alias(void) +{ + TEST("read-only MAP_SHARED mremap does not flush writable alias"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-readonly-alias-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (2 * span)) != 0 || pwrite(fd, "F", 1, 0) != 1) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *writer = + map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, 0); + char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0); + if (writer == MAP_FAILED || source == MAP_FAILED) { + FAIL("snapshot MAP_SHARED mappings failed"); + if (writer != MAP_FAILED) + munmap(writer, span); + if (source != MAP_FAILED) + munmap(source, span); + close(fd); + return; + } + writer[0] = 'W'; + + void *blocker = mmap(source + span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != source + span) { + FAIL("mremap blocker mmap failed"); + munmap(writer, span); + munmap(source, span); + close(fd); + return; + } + + char *moved = mremap(source, span, 2 * span, MREMAP_MAYMOVE); + unsigned char file_byte = 0; + bool unchanged = moved != MAP_FAILED && moved != source && + pread(fd, &file_byte, 1, 0) == 1 && file_byte == 'F'; + if (unchanged) + PASS(); + else + FAIL("read-only source mremap flushed writable alias"); + + if (moved != MAP_FAILED) + munmap(moved, 2 * span); + else + munmap(source, span); + munmap(writer, span); + munmap(blocker, span); + close(fd); +} + static void test_file_backed_fork_split_move(void) { TEST("MAP_SHARED file: mremap across fork-split source"); @@ -977,6 +1058,7 @@ int main(void) test_postfork_repeated_allocations_keep_lineage(); test_adjacent_file_vmas_rejected(); test_misaligned_shared_mremap_writeback(); + test_readonly_shared_mremap_does_not_flush_alias(); test_file_backed_fork_split_move(); test_file_backed_mprotect_fragments_move(); test_file_backed_fixed_move_from_same_vma(); diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c index 0096c443..d39551b8 100644 --- a/tests/test-proc-smap.c +++ b/tests/test-proc-smap.c @@ -32,10 +32,12 @@ typedef struct { unsigned long long mmu_page_kb; unsigned long long rss_kb; unsigned long long pss_kb; + unsigned long long pss_dirty_kb; unsigned long long shared_dirty_kb; bool have_size; bool have_rss; bool have_pss; + bool have_pss_dirty; bool have_shared_dirty; bool have_vmflags; bool vmflags_wr; @@ -170,10 +172,12 @@ static bool parse_header(const char *line, smaps_vma_t *vma) vma->mmu_page_kb = 0; vma->rss_kb = 0; vma->pss_kb = 0; + vma->pss_dirty_kb = 0; vma->shared_dirty_kb = 0; vma->have_size = false; vma->have_rss = false; vma->have_pss = false; + vma->have_pss_dirty = false; vma->have_shared_dirty = false; vma->have_vmflags = false; vma->vmflags_wr = false; @@ -246,8 +250,8 @@ static bool parse_vmflags(const char *line, bool *writable) static bool finish_vma(smaps_vma_t *vma, size_t field_index) { return field_index == SMAPS_FIELD_COUNT && vma->have_size && - vma->have_rss && vma->have_pss && vma->have_shared_dirty && - vma->have_vmflags; + vma->have_rss && vma->have_pss && vma->have_pss_dirty && + vma->have_shared_dirty && vma->have_vmflags; } static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) @@ -319,6 +323,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.rss_kb = value; if (field_index == 4) current.pss_kb = value; + if (field_index == 5) + current.pss_dirty_kb = value; if (field_index == 7) current.shared_dirty_kb = value; if (field_index == 0) @@ -327,6 +333,8 @@ static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) current.have_rss = true; if (field_index == 4) current.have_pss = true; + if (field_index == 5) + current.have_pss_dirty = true; if (field_index == 7) current.have_shared_dirty = true; field_index++; @@ -436,9 +444,12 @@ static bool validate_layout(const smaps_info_t *info, last->mmu_page_kb != 4) return false; if (first->rss_kb != first->shared_dirty_kb || - first->pss_kb != first->shared_dirty_kb || middle->rss_kb != 0 || - middle->pss_kb != 0 || last->rss_kb != last->shared_dirty_kb || - last->pss_kb != last->shared_dirty_kb) + first->pss_kb != first->shared_dirty_kb || + first->pss_dirty_kb != first->shared_dirty_kb || middle->rss_kb != 0 || + middle->pss_kb != 0 || middle->pss_dirty_kb != 0 || + last->rss_kb != last->shared_dirty_kb || + last->pss_kb != last->shared_dirty_kb || + last->pss_dirty_kb != last->shared_dirty_kb) return false; /* Every stress-map page has deliberately distinct permissions (with one @@ -460,7 +471,8 @@ static bool validate_layout(const smaps_info_t *info, const smaps_vma_t *stress_none = find_vma(info, stress + 2 * page_size); if (!stress_none || strcmp(stress_none->perms, "---p") || stress_none->vmflags_wr || stress_none->shared_dirty_kb != 0 || - stress_none->rss_kb != 0 || stress_none->pss_kb != 0) + stress_none->rss_kb != 0 || stress_none->pss_kb != 0 || + stress_none->pss_dirty_kb != 0) return false; return true; } @@ -568,11 +580,16 @@ static int child_probe(uintptr_t target, const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || !stress_none || !postfork_vma || first->shared_dirty_kb == 0 || + first->pss_dirty_kb != first->shared_dirty_kb || middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || + last->pss_dirty_kb != last->shared_dirty_kb || stress_ro->shared_dirty_kb != 0 || stress_rw->shared_dirty_kb == 0 || + stress_rw->pss_dirty_kb != stress_rw->shared_dirty_kb || stress_none->shared_dirty_kb != 0 || stress_none->rss_kb != 0 || - stress_none->pss_kb != 0 || postfork_vma->shared_dirty_kb != 0) { + stress_none->pss_kb != 0 || stress_none->pss_dirty_kb != 0 || + postfork_vma->shared_dirty_kb != 0 || + postfork_vma->pss_dirty_kb != 0) { free_smaps(&info); munmap(postfork, page_size); return 1; From 76a80bd8651768f507d94c59e1891c3497a64f1a Mon Sep 17 00:00:00 2001 From: Xiaoguang Sun Date: Mon, 3 Aug 2026 16:37:46 +0800 Subject: [PATCH 14/14] Preserve failed mmap reservations Keep failed reservation addresses visible to callers when munmap fails. This lets MAP_FAILED cleanup release still-live mappings instead of leaking them. --- tests/test-mremap-fork-tracking.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c index 7649a180..10c2175a 100644 --- a/tests/test-mremap-fork-tracking.c +++ b/tests/test-mremap-fork-tracking.c @@ -469,17 +469,25 @@ static void test_misaligned_shared_mremap_writeback(void) static void *map_misaligned_shared_fixed(size_t length, int prot, int fd, - off_t offset) + off_t offset, + void **reservation_out) { const size_t page = 4096; + *reservation_out = NULL; + void *reservation = mmap(NULL, length + page, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (reservation == MAP_FAILED) return MAP_FAILED; void *address = (char *) reservation + page; - if (munmap(reservation, length + page) != 0) + if (munmap(reservation, length + page) != 0) { + /* Keep ownership visible to the caller when the reservation cannot + * be released. Returning MAP_FAILED alone would make the live + * mapping indistinguishable from an allocation that never happened. */ + *reservation_out = reservation; return MAP_FAILED; + } return mmap(address, length, prot, MAP_SHARED | MAP_FIXED, fd, offset); } @@ -487,6 +495,7 @@ static void test_readonly_shared_mremap_does_not_flush_alias(void) { TEST("read-only MAP_SHARED mremap does not flush writable alias"); + const size_t page = 4096; const size_t span = 64 * 1024; char tmpl[] = "/tmp/elfuse-mremap-readonly-alias-XXXXXX"; int fd = mkstemp(tmpl); @@ -501,15 +510,22 @@ static void test_readonly_shared_mremap_does_not_flush_alias(void) return; } - char *writer = - map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, 0); - char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0); + void *writer_reservation = NULL; + void *source_reservation = NULL; + char *writer = map_misaligned_shared_fixed(span, PROT_READ | PROT_WRITE, fd, + 0, &writer_reservation); + char *source = map_misaligned_shared_fixed(span, PROT_READ, fd, 0, + &source_reservation); if (writer == MAP_FAILED || source == MAP_FAILED) { FAIL("snapshot MAP_SHARED mappings failed"); if (writer != MAP_FAILED) munmap(writer, span); if (source != MAP_FAILED) munmap(source, span); + if (writer_reservation != NULL) + munmap(writer_reservation, span + page); + if (source_reservation != NULL) + munmap(source_reservation, span + page); close(fd); return; }