diff --git a/Makefile b/Makefile index 02549c93..2ce3816c 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,8 @@ include mk/config.mk # Source files. SRCS := \ main.c \ + dynamic-array.c \ + string-builder.c \ core/guest.c \ core/elf.c \ core/stack.c \ @@ -192,6 +194,23 @@ $(BUILD_DIR)/test-teardown-live-vcpu-host: \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ $(HVF_LDFLAGS) +## Build the string builder host unit test (native macOS binary) +# This is a pure-C unit test; link the string builder and generic container +# implementations directly and skip the Hypervisor framework and codesign. +$(BUILD_DIR)/test-string-builder-host: \ + $(BUILD_DIR)/test-string-builder-host.o \ + $(BUILD_DIR)/string-builder.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the generic dynamic-array host unit test (native macOS binary) +$(BUILD_DIR)/test-dynamic-array-host: \ + $(BUILD_DIR)/test-dynamic-array-host.o \ + $(BUILD_DIR)/dynamic-array.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + # Guest test binaries (cross-compiled, aarch64-linux) # Only used when GUEST_TEST_BINARIES is not set. @@ -356,6 +375,11 @@ endif endif +## Build the libc-based file-backed mremap EMFILE regression probe +$(BUILD_DIR)/test-mremap-tail-emfile: tests/test-mremap-tail-emfile.c | $(BUILD_DIR) + @echo " CROSS $<" + $(Q)$(CROSS_COMPILE)gcc -D_GNU_SOURCE -static -O2 -o $@ $< + include mk/tests.mk include mk/analysis.mk include mk/help.mk diff --git a/docs/internals.md b/docs/internals.md index e6732d48..3096dc26 100644 --- a/docs/internals.md +++ b/docs/internals.md @@ -114,6 +114,32 @@ Key files: | `src/runtime/proctitle.c` | argv / comm rewriting for `prctl PR_SET_NAME` | | `src/debug/gdbstub.c`, `gdbstub-rsp.c`, `gdbstub-reg.c` | GDB RSP stub | +## Generic Dynamic Containers + +`src/dynamic-array.h` and `src/dynamic-array.c` provide the raw +`dynamic_array_t` used by the procfs VMA snapshot and the string builder. Capacity +is measured in element slots, while `count` is the number of logical elements. +The allocation is one contiguous block of `capacity * element_size` bytes; +both the count addition and the multiplication are checked before a growth. +Arithmetic overflow reports `EOVERFLOW`, invalid arguments report `EINVAL`, +and an allocation failure reports `ENOMEM`. Growth is transactional: on any +failure the old pointer, count, and capacity remain valid. + +The generated typed facades own only their array storage. Elements are copied +as trivially-copyable bytes, so the container does not call destructors and +does not manage pointers or other resources held by an element. `destroy` +frees the contiguous block and restores the zero state. A facade can therefore +be declared as `{0}` and initialized lazily on its first operation. + +`string_builder_t` is a thin facade over a generated `char` container. The +container count is the C-string length and excludes the terminator; its +capacity accessor reports bytes including the trailing NUL slot. Reserve and +append account for that slot, and every successful mutation restores +`data[count] == '\0'`. `string_builder_append` accepts a C string and uses its +first NUL as the end of the input. Formatted appends retain the existing +two-pass `vsnprintf` behavior and commit only the prefix through the first NUL, +matching standard C string semantics. + ## Hypervisor.framework Constraints Apple HVF imposes a handful of constraints that shape the rest of the design: @@ -768,6 +794,53 @@ under `/proc`, `/dev`, and a few Linux-expected compatibility files: - Guest cwd handling preserves a virtual `/proc` working directory even though the host operates on synthetic backing directories. +`/proc/self/smaps` and `/proc//smaps` are generated from the same tracked +VMA list as `/proc/self/maps`. The complete field set currently emitted for +each VMA is the maps header followed by these 25 fields, in this order: + +```text +Size, KernelPageSize, MMUPageSize, Rss, Pss, Pss_Dirty, +Shared_Clean, Shared_Dirty, Private_Clean, Private_Dirty, Referenced, +Anonymous, KSM, LazyFree, AnonHugePages, ShmemPmdMapped, FilePmdMapped, +Shared_Hugetlb, Private_Hugetlb, Swap, SwapPss, Locked, THPeligible, +ProtectionKey, VmFlags +``` + +`Size` is the VMA length in KiB. `KernelPageSize` and `MMUPageSize` are +reported as 4 KiB. `Shared_Dirty` is the one page-accounting field with a +non-zero value: in a fork child, writable private anonymous VMAs that existed +in the parent's CoW snapshot report their full VMA size because they are +logically shared with that snapshot. Every other numeric counter, including +`THPeligible` and `ProtectionKey`, is emitted as zero. `VmFlags` is +evidence-based and contains only the permission/sharing flags represented by +the tracked VMA (`rd`, `wr`, +`ex`, `sh`, and `nr` when applicable); no untracked kernel flags are invented. +elfuse always emits `THPeligible` and `ProtectionKey`; consumers comparing +against a real Linux kernel should tolerate either conditional field being +absent. +These are coarse VMA-level values, not host page-residency, dirty-bit, or +proportional-sharing accounting, so they are suitable for fork-safety checks +but not precise memory profiling. The fork-child marker is tracked per VMA, so +writable private anonymous mappings created after fork are excluded from the +compatibility signal. `smaps_rollup` is not implemented. + +Synthetic proc directories have explicit snapshot boundaries. The backing +trees reached by opening `/proc` or `/proc/self` are materialized once, on the +first access, and their initial `stat`, `status`, `cmdline`, `maps`, and `smaps` +files remain fixed for the process lifetime. An absolute open of one of those +proc paths is intercepted directly and generates fresh content; opening the +same name through an already-open synthetic directory reads that directory's +snapshot. `/proc/self/fd` and `/proc/self/fdinfo` are rebuilt into an +independent scratch directory on every open, so each directory fd sees the +guest-fd table as it existed at that open and concurrent enumerations cannot +mutate one another. `/proc/self/task` is repopulated from the current thread +set whenever the directory is opened. These boundaries are intentional: a +directory stream is stable while it is being read, while dynamic task and fd +listings refresh only when a new directory is opened. The synthetic +`/sys/devices/system/cpu` tree follows the one-shot rule: its CPU count, +cpumask files, and `cpuN` directories are captured on first access and then +remain fixed. + Related implementation: `src/runtime/procemu.c`, `src/syscall/path.c`, `src/syscall/fs.c`, `src/syscall/proc-state.c`. diff --git a/docs/testing.md b/docs/testing.md index c42e1b72..fb8a4cb4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -92,7 +92,8 @@ What they do: - the unit suite from `tests/manifest.txt` -- deliberately narrow: only tests that assert elfuse-internal implementation details with no real Linux counterpart (the EL1 shim fast-path suite, `test-mremap-infra`, - `test-oom-proc`), plus whatever `mk/tests.mk`'s `SANITIZER_SECTIONS` + `test-mremap-fork-tracking`, `test-oom-proc`), plus whatever + `mk/tests.mk`'s `SANITIZER_SECTIONS` needs for the `check-{asan,ubsan,tsan}` lanes. Everything that is meaningful to cross-check against a real Linux kernel lives exclusively in `tests/test-matrix.sh`'s `run_unit_tests` instead (see Test Matrix @@ -168,8 +169,9 @@ surface -- every binary that is meaningful to run against a real kernel, which is almost everything. It deliberately excludes only the handful of tests that assert elfuse-internal implementation details with no meaningful counterpart on a real kernel (the EL1 shim fast-path suite, `test-mremap-infra`, -`test-oom-proc` -- these live solely in `tests/manifest.txt` / `make check`, -see that file's header for the full split rationale). There is no separate +`test-mremap-fork-tracking`, `test-oom-proc` -- these live solely in +`tests/manifest.txt` / `make check`, see that file's header for the full split +rationale). There is no separate "core" vs "extended" test set inside the matrix; a test that has a real, understood divergence from the qemu reference kernel is listed in `QEMU_SKIP` with a comment explaining why instead -- see that variable in diff --git a/mk/config.mk b/mk/config.mk index 969a39fc..c53ac42b 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -21,7 +21,9 @@ endif # Exclude native macOS test files from cross-compilation NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c \ tests/test-tlbi-encoder-host.c \ - tests/test-fork-ipc-protocol-host.c + tests/test-fork-ipc-protocol-host.c \ + tests/test-dynamic-array-host.c \ + tests/test-string-builder-host.c SPECIAL_TEST_SRCS := tests/test-lowbase-mem.c SPECIAL_TEST_BINS := $(BUILD_DIR)/test-lowbase-mem-200000 $(BUILD_DIR)/test-lowbase-mem-300000 diff --git a/mk/tests.mk b/mk/tests.mk index 3cbc1c58..f879948d 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -14,6 +14,9 @@ test-sysroot-host-fallback test-sysroot-case-exact \ test-sysroot-create-paths test-fork-ipc-protocol-host \ test-vcpu-run-hooks-host test-identity-override-host \ + test-dynamic-array-host \ + test-string-builder-host \ + test-mremap-tail-emfile \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-fuse-alpine \ test-sysroot-nofollow test-sysroot-chdir test-sysroot-symlink-escape \ @@ -24,6 +27,11 @@ test-hello: $(ELFUSE_BIN) $(TEST_HELLO_DEP) @printf "$(BLUE)▸ Running$(RESET) test-hello\n" $(ELFUSE_BIN) $(TEST_DIR)/test-hello +## Run the libc-based file-backed region removal EMFILE regression probe +test-mremap-tail-emfile: $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + @printf "$(BLUE)▸ Running$(RESET) test-mremap-tail-emfile\n" + $(ELFUSE_BIN) $(BUILD_DIR)/test-mremap-tail-emfile + ## Verify dispatch.tbl coverage of the kernel-supported syscall set check-syscall-coverage: @python3 scripts/check-syscall-coverage.py @@ -80,7 +88,9 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ $(BUILD_DIR)/test-fork-ipc-protocol-host \ $(BUILD_DIR)/test-vcpu-run-hooks-host \ $(BUILD_DIR)/test-identity-override-host \ - $(BUILD_DIR)/test-teardown-live-vcpu-host + $(BUILD_DIR)/test-teardown-live-vcpu-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v -s '$(SANITIZER_SECTIONS)' @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -92,6 +102,10 @@ check-sanitizer: $(ELFUSE_BIN) $(TEST_DEPS) \ @$(BUILD_DIR)/test-identity-override-host @printf "\n$(BLUE)━━━ teardown live-worker accounting unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-teardown-live-vcpu-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host ## Run the unit test suite plus busybox applet validation check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @@ -99,7 +113,9 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ $(BUILD_DIR)/test-fork-ipc-protocol-host \ $(BUILD_DIR)/test-vcpu-run-hooks-host \ $(BUILD_DIR)/test-identity-override-host \ - $(BUILD_DIR)/test-teardown-live-vcpu-host + $(BUILD_DIR)/test-teardown-live-vcpu-host \ + $(BUILD_DIR)/test-dynamic-array-host \ + $(BUILD_DIR)/test-string-builder-host @bash tests/driver.sh -e $(ELFUSE_BIN) -d $(TEST_DIR) -v @printf "\n$(BLUE)━━━ TLBI RVAE1IS encoder unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-tlbi-encoder-host @@ -111,6 +127,10 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage \ @$(BUILD_DIR)/test-identity-override-host @printf "\n$(BLUE)━━━ teardown live-worker accounting unit test ━━━$(RESET)\n" @$(BUILD_DIR)/test-teardown-live-vcpu-host + @printf "\n$(BLUE)━━━ dynamic array unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-dynamic-array-host + @printf "\n$(BLUE)━━━ string builder unit test ━━━$(RESET)\n" + @$(BUILD_DIR)/test-string-builder-host @printf "\n$(BLUE)━━━ shebang parser unit test ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-shebang-host @printf "\n$(BLUE)━━━ proctitle argv-tail regression ━━━$(RESET)\n" @@ -770,6 +790,16 @@ test-fork-ipc-protocol-host: $(BUILD_DIR)/test-fork-ipc-protocol-host test-vcpu-run-hooks-host: $(BUILD_DIR)/test-vcpu-run-hooks-host $(BUILD_DIR)/test-vcpu-run-hooks-host +# String builder unit test +## Run the growable string builder host unit test +test-string-builder-host: $(BUILD_DIR)/test-string-builder-host + $(BUILD_DIR)/test-string-builder-host + +# Generic dynamic array unit test +## Run the raw/typed dynamic array host unit test +test-dynamic-array-host: $(BUILD_DIR)/test-dynamic-array-host + $(BUILD_DIR)/test-dynamic-array-host + # Proctitle argv-tail regression ## Run the deterministic argv-tail overshoot guard test test-proctitle-host: $(BUILD_DIR)/test-proctitle-host diff --git a/src/core/guest.c b/src/core/guest.c index 2bf395e0..7620dcb2 100644 --- a/src/core/guest.c +++ b/src/core/guest.c @@ -1840,15 +1840,18 @@ int guest_get_used_regions(const guest_t *g, /* Semantic region tracking. * - * Check whether two adjacent regions can be merged. They must be contiguous in - * address space, have identical protection/flags/name, and have contiguous file - * offsets (so the merged region still represents valid mapping). For anonymous - * regions the offset is meaningless (always 0, but may become non-zero after - * split/trim), so the contiguity check is skipped. Without this, adjacent - * anonymous mmaps (common in megablock-style allocators) each create separate - * entries that exhaust the region table. + * Check whether two adjacent regions have merge-compatible layouts. An actual + * merge additionally requires a shared vma_id; a new same-generation mapping + * may adopt its compatible neighbor's ID below. Regions must be contiguous in + * address space, have identical protection/flags/name, and have contiguous + * file offsets (so the merged region still represents valid mapping). For + * anonymous regions the offset is meaningless (always 0, but may become + * non-zero after split/trim), so the contiguity check is skipped. Without this, + * adjacent anonymous mmaps (common in megablock-style allocators) each create + * separate entries that exhaust the region table. */ -static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +static bool regions_mergeable_layout(const guest_region_t *a, + const guest_region_t *b) { if (a->end != b->start) return false; @@ -1870,6 +1873,8 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return false; if (a->backing_ro != b->backing_ro) return false; + if (a->inherited_at_fork != b->inherited_at_fork) + return false; if (a->overlay_active || b->overlay_active) return false; if (strcmp(a->name, b->name) != 0) @@ -1884,6 +1889,11 @@ static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) return a->offset + (a->end - a->start) == b->offset; } +static bool regions_mergeable(const guest_region_t *a, const guest_region_t *b) +{ + return a->vma_id == b->vma_id && regions_mergeable_layout(a, b); +} + /* First region whose start is >= start. regions[] is sorted by start. */ static int region_lower_bound_start(const guest_t *g, uint64_t start) { @@ -1981,7 +1991,8 @@ int guest_region_add_ex(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false, + 0); } int guest_region_add_ex_gpa(guest_t *g, @@ -2002,7 +2013,31 @@ int guest_region_add_ex_gpa(guest_t *g, } return guest_region_add_ex_owned_gpa(g, start, end, gpa_base, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, false, + 0); +} + +static uint64_t allocate_vma_id(guest_t *g) +{ + uint64_t candidate = g->next_vma_id; + + for (;;) { + candidate++; + if (candidate == 0) + candidate = 1; + + bool in_use = false; + for (int i = 0; i < g->nregions; i++) { + if (g->regions[i].vma_id == candidate) { + in_use = true; + break; + } + } + if (!in_use) { + g->next_vma_id = candidate; + return candidate; + } + } } int guest_region_add_ex_owned(guest_t *g, @@ -2012,10 +2047,13 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id) { return guest_region_add_ex_owned_gpa(g, start, end, start, prot, flags, - offset, name, owned_backing_fd); + offset, name, owned_backing_fd, + inherited_at_fork, vma_id); } int guest_region_add_ex_owned_gpa(guest_t *g, @@ -2026,7 +2064,9 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd) + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id) { if (g->nregions >= GUEST_MAX_REGIONS) { log_error( @@ -2039,6 +2079,12 @@ int guest_region_add_ex_owned_gpa(guest_t *g, return -1; } + bool new_vma = !vma_id; + if (new_vma) + vma_id = allocate_vma_id(g); + else if (vma_id > g->next_vma_id) + g->next_vma_id = vma_id; + /* Find insertion point (keep sorted by start address). */ int i = region_lower_bound_start(g, start); memmove(&g->regions[i + 1], &g->regions[i], @@ -2048,13 +2094,15 @@ int guest_region_add_ex_owned_gpa(guest_t *g, r->start = start; r->end = end; r->gpa_base = gpa_base; + r->vma_id = vma_id; r->prot = prot; r->flags = flags; r->offset = offset; r->backing_fd = owned_backing_fd; - r->shared = (flags & 0x01) != 0; /* LINUX_MAP_SHARED = 0x01 */ - r->noreserve = (flags & 0x4000) != 0; /* LINUX_MAP_NORESERVE = 0x4000 */ + r->shared = (flags & LINUX_MAP_SHARED) != 0; + r->noreserve = (flags & LINUX_MAP_NORESERVE) != 0; r->backing_ro = false; + r->inherited_at_fork = inherited_at_fork; guest_region_clear_overlay(r); if (name) { str_copy_trunc(r->name, name, sizeof(r->name)); @@ -2063,6 +2111,20 @@ int guest_region_add_ex_owned_gpa(guest_t *g, } g->nregions++; + /* Preserve the historical coalescing of compatible same-generation + * anonymous mmap calls: Linux may merge those into one VMA and the region + * tracker relies on that to stay below GUEST_MAX_REGIONS. Never adopt a + * neighbor's lineage across an inherited/private boundary, which is the + * provenance distinction find_mremap_source() must retain after fork. + */ + if (new_vma) { + if (i > 0 && regions_mergeable_layout(&g->regions[i - 1], r)) + r->vma_id = g->regions[i - 1].vma_id; + else if (i + 1 < g->nregions && + regions_mergeable_layout(r, &g->regions[i + 1])) + r->vma_id = g->regions[i + 1].vma_id; + } + /* Try to merge with adjacent regions to reduce table pressure. Merge right * first, then left (order matters: right merge does not change the index of * the left neighbor). @@ -2105,10 +2167,44 @@ int guest_preannounce(guest_t *g, return 0; } -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd) { + if (!reserved_backing_fd) + return -1; + *reserved_backing_fd = -1; if (end <= start) - return; + return 0; + + int first = guest_region_first_end_above(g, start); + for (int i = first; i < g->nregions; i++) { + const guest_region_t *r = &g->regions[i]; + if (r->start >= end) + break; + if (r->start < start && r->end > end) { + /* A full table follows the existing stale-tracker fallback and + * does not publish a right-hand record, so no fd is required. */ + if (g->nregions >= GUEST_MAX_REGIONS || r->backing_fd < 0) + return 0; + *reserved_backing_fd = dup(r->backing_fd); + return *reserved_backing_fd >= 0 ? 0 : -1; + } + } + return 0; +} + +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd) +{ + if (end <= start) { + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; + } /* In-place compaction: 'out' is the next output slot, 'in' is the next * input slot. Since the prefix [0, first) is untouched (it sorts strictly @@ -2168,18 +2264,16 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) right.gpa_base += trimmed; right.start = end; if (orig.backing_fd >= 0) { - right.backing_fd = dup(orig.backing_fd); - if (right.backing_fd < 0) - log_error( - "guest: dup() failed for region split " - "backing fd %d: %s", - orig.backing_fd, strerror(errno)); + right.backing_fd = reserved_backing_fd; + reserved_backing_fd = -1; } guest_region_clip_overlay(&right); g->regions[out + 1] = right; g->nregions = out + 2 + suffix_count; - return; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; } } @@ -2217,6 +2311,17 @@ void guest_region_remove(guest_t *g, uint64_t start, uint64_t end) memmove(&g->regions[out], &g->regions[in], tail * sizeof(guest_region_t)); g->nregions = out + tail; + if (reserved_backing_fd >= 0) + close(reserved_backing_fd); + return 0; +} + +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end) +{ + int reserved_backing_fd = -1; + if (guest_region_remove_prepare(g, start, end, &reserved_backing_fd) < 0) + return -1; + return guest_region_remove_reserved(g, start, end, reserved_backing_fd); } const guest_region_t *guest_region_find(const guest_t *g, uint64_t addr) diff --git a/src/core/guest.h b/src/core/guest.h index 5e80d9ee..0645e70b 100644 --- a/src/core/guest.h +++ b/src/core/guest.h @@ -230,6 +230,12 @@ typedef struct { * identity-mapped regions; differs for high-VA guest * mappings whose VA and GPA diverge. */ + uint64_t vma_id; /* Stable logical-VMA lineage. Tracker splits and + * mremap moves preserve it; compatible same-generation + * mappings may share it when the tracker coalesces + * them. Unlike inherited_at_fork, this ID remains + * meaningful across subsequent forks. + */ int prot; /* LINUX_PROT_* flags */ int flags; /* LINUX_MAP_* flags (for /proc/self/maps display) */ uint64_t offset; /* File offset (for /proc/self/maps display) */ @@ -242,6 +248,11 @@ typedef struct { * later PROT_WRITE request against it with EACCES, * matching a real kernel's VMA max_prot tracking. */ + bool inherited_at_fork; /* Region existed at the most recent fork + * snapshot. Used by synthetic smaps to report + * only VMAs that can participate in that fork's + * CoW snapshot as Shared_Dirty. + */ bool overlay_active; /* Region has a live host MAP_FIXED|MAP_SHARED overlay * of backing_fd at host_base+start. The kernel's page * cache keeps it coherent with the file and with peer @@ -503,7 +514,8 @@ typedef struct { /* Semantic region tracking for munmap/mprotect/proc-self-maps */ guest_region_t regions[GUEST_MAX_REGIONS]; - int nregions; /* Number of active regions */ + int nregions; /* Number of active regions */ + uint64_t next_vma_id; /* Last logical-VMA lineage ID allocated. */ /* Sticky flag set when guest_region_set_prot could not honor a request * because the region table was full. After this point the tracker no longer * faithfully reflects PTE state, so the mprotect fast path must fall back @@ -1176,7 +1188,9 @@ int guest_region_add_ex_gpa(guest_t *g, int backing_fd); /* Like guest_region_add_ex, but consumes owned_backing_fd on success or - * failure. + * failure. inherited_at_fork identifies bytes copied from the latest fork + * snapshot. vma_id preserves logical-VMA provenance across tracker splits and + * mremap moves; pass 0 for a new VMA. */ int guest_region_add_ex_owned(guest_t *g, uint64_t start, @@ -1185,7 +1199,9 @@ int guest_region_add_ex_owned(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id); int guest_region_add_ex_owned_gpa(guest_t *g, uint64_t start, uint64_t end, @@ -1194,7 +1210,9 @@ int guest_region_add_ex_owned_gpa(guest_t *g, int flags, uint64_t offset, const char *name, - int owned_backing_fd); + int owned_backing_fd, + bool inherited_at_fork, + uint64_t vma_id); /* Add a preannounced region that appears in /proc/self/maps only. These entries * are kept separate from regions[] so they do not cause -EEXIST on guest @@ -1214,10 +1232,29 @@ int guest_preannounce(guest_t *g, uint64_t offset, const char *name); +/* Reserve any backing fd needed by an interior split in [start, end). + * The reservation must be consumed by guest_region_remove_reserved(). + * Returns 0 on success, -1 when the backing fd cannot be duplicated. + */ +int guest_region_remove_prepare(guest_t *g, + uint64_t start, + uint64_t end, + int *reserved_backing_fd); + /* Remove all region coverage in [start, end). Regions fully contained are - * deleted; partially overlapping regions are trimmed or split. + * deleted; partially overlapping regions are trimmed or split. Any required + * backing fd is reserved before the first metadata mutation. + */ +int guest_region_remove(guest_t *g, uint64_t start, uint64_t end); + +/* Commit a removal using a backing fd reserved by + * guest_region_remove_prepare(). Ownership of reserved_backing_fd is consumed + * even when this call does not need an interior split. */ -void guest_region_remove(guest_t *g, uint64_t start, uint64_t end); +int guest_region_remove_reserved(guest_t *g, + uint64_t start, + uint64_t end, + int reserved_backing_fd); /* Find the region containing addr. * diff --git a/src/dynamic-array.c b/src/dynamic-array.c new file mode 100644 index 00000000..50db79c9 --- /dev/null +++ b/src/dynamic-array.c @@ -0,0 +1,336 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "dynamic-array.h" + +#include +#include +#include +#include + +#define DYNAMIC_ARRAY_INITIAL_CAPACITY ((size_t) 8) + +/* Set errno for a bad argument and return a standard failure code. */ +static int dynamic_array_invalid(void) +{ + errno = EINVAL; + return -1; +} + +/* Check that the array handle is non-null and has a non-zero element size. */ +static int dynamic_array_validate(const dynamic_array_t *array) +{ + if (array == NULL || array->element_size == 0) + return dynamic_array_invalid(); + return 0; +} + +/* Return count + extra after guarding against size_t overflow. */ +static int dynamic_array_count_plus(const dynamic_array_t *array, + size_t extra, + size_t *total) +{ + if (extra > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + *total = array->count + extra; + return 0; +} + +/* Compute the byte size for a number of elements with overflow protection. */ +static int dynamic_array_bytes(const dynamic_array_t *array, + size_t count, + size_t *bytes) +{ + if (count != 0 && array->element_size > SIZE_MAX / count) { + errno = EOVERFLOW; + return -1; + } + *bytes = count * array->element_size; + return 0; +} + +/* Return the offset of an in-array source span, including capacity bytes. */ +static int dynamic_array_source_offset(const dynamic_array_t *array, + const void *source, + size_t bytes, + size_t *offset) +{ + if (array->data == NULL || source == NULL) + return 0; + + uintptr_t base = (uintptr_t) array->data; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t) SIZE_MAX) + return 0; + size_t start = (size_t) delta; + size_t allocation_bytes; + if (dynamic_array_bytes(array, array->capacity, &allocation_bytes) < 0) + return 0; + if (start > allocation_bytes || bytes > allocation_bytes - start) + return 0; + *offset = start; + return 1; +} + +/* Initialize only metadata; allocate storage separately when requested. + * + * This is a fresh-initialization operation and deliberately does not inspect + * prior object contents, so an automatic, uninitialized object is safe. Use + * dynamic_array_destroy before reinitializing an array that already owns + * storage; otherwise that allocation is intentionally abandoned. + */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + *array = (dynamic_array_t) { + .element_size = element_size, + }; + return 0; +} + +/* Initialize and reserve storage for an initial number of elements. + * + * Like dynamic_array_init, this is a fresh-initialization operation that does + * not inspect prior object contents. In particular, do not free an + * indeterminate pointer from an automatic object; destroy an existing array + * before reinitializing it. + */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity) +{ + if (array == NULL || element_size == 0) + return dynamic_array_invalid(); + + /* Establish a safe zero state before any fallible allocation. This is a + * fresh initializer, so an existing allocation must have been destroyed + * by the caller rather than silently leaked here. */ + *array = (dynamic_array_t) {0}; + + size_t bytes; + if (initial_capacity != 0 && element_size > SIZE_MAX / initial_capacity) { + errno = EOVERFLOW; + return -1; + } + bytes = initial_capacity * element_size; + void *storage = NULL; + if (bytes != 0) { + storage = malloc(bytes); + if (storage == NULL) { + errno = ENOMEM; + return -1; + } + } + + *array = (dynamic_array_t) { + .data = storage, + .capacity = initial_capacity, + .element_size = element_size, + }; + return 0; +} + +/* Release backing storage and reset the array object to zero state. */ +void dynamic_array_destroy(dynamic_array_t *array) +{ + if (array == NULL) + return; + free(array->data); + *array = (dynamic_array_t) {0}; +} + +/* Ensure the array has capacity for at least extra additional elements. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra) +{ + if (dynamic_array_validate(array) < 0) + return -1; + + size_t needed; + if (dynamic_array_count_plus(array, extra, &needed) < 0) + return -1; + if (needed <= array->capacity) + return 0; + + size_t new_capacity = array->capacity; + if (new_capacity == 0) + new_capacity = DYNAMIC_ARRAY_INITIAL_CAPACITY; + while (new_capacity < needed) { + if (new_capacity > SIZE_MAX / 2) { + new_capacity = needed; + break; + } + new_capacity *= 2; + } + + size_t bytes; + if (dynamic_array_bytes(array, new_capacity, &bytes) < 0) + return -1; + void *grown = realloc(array->data, bytes); + if (grown == NULL && bytes != 0) { + errno = ENOMEM; + return -1; + } + array->data = grown; + array->capacity = new_capacity; + return 0; +} + +/* Resize logical length; zero-initialize any newly visible elements. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count > array->capacity) { + size_t extra = count - array->count; + if (dynamic_array_reserve(array, extra) < 0) + return -1; + } + if (count > array->count) { + size_t old_bytes, new_bytes; + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; + if (dynamic_array_bytes(array, count, &new_bytes) < 0) + return -1; + memset((unsigned char *) array->data + old_bytes, 0, + new_bytes - old_bytes); + } + array->count = count; + return 0; +} + +/* Append multiple elements from source memory to the end of the array. */ +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, + size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (dynamic_array_count_plus(array, count, &total) < 0) + return -1; + size_t bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0) + return -1; + size_t offset = 0; + int aliases = dynamic_array_source_offset(array, data, bytes, &offset); + + if (dynamic_array_reserve(array, count) < 0) + return -1; + if (aliases) + data = (const unsigned char *) array->data + offset; + size_t old_bytes; + if (dynamic_array_bytes(array, array->count, &old_bytes) < 0) + return -1; + memmove((unsigned char *) array->data + old_bytes, data, bytes); + array->count = total; + return 0; +} + +/* Insert multiple elements at index while preserving existing elements. */ +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count) +{ + if (dynamic_array_validate(array) < 0) + return -1; + if (index > array->count) + return dynamic_array_invalid(); + if (count == 0) + return 0; + if (data == NULL) + return dynamic_array_invalid(); + + size_t total; + if (count > SIZE_MAX - array->count) { + errno = EOVERFLOW; + return -1; + } + total = array->count + count; + size_t bytes, index_bytes, tail_bytes; + if (dynamic_array_bytes(array, count, &bytes) < 0 || + dynamic_array_bytes(array, index, &index_bytes) < 0 || + dynamic_array_bytes(array, array->count - index, &tail_bytes) < 0) + return -1; + + size_t source_offset = 0; + int aliases = + dynamic_array_source_offset(array, data, bytes, &source_offset); + void *temporary = NULL; + if (aliases) { + temporary = malloc(bytes); + if (temporary == NULL) { + errno = ENOMEM; + return -1; + } + memcpy(temporary, (const unsigned char *) array->data + source_offset, + bytes); + data = temporary; + } + + if (dynamic_array_reserve(array, count) < 0) { + free(temporary); + return -1; + } + unsigned char *base = array->data; + memmove(base + index_bytes + bytes, base + index_bytes, tail_bytes); + memcpy(base + index_bytes, data, bytes); + array->count = total; + free(temporary); + return 0; +} + +/* Append a single element by forwarding to append_n. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data) +{ + return dynamic_array_append_n(array, data, 1); +} + +/* Insert a single element by forwarding to insert_n. */ +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, + const void *data) +{ + return dynamic_array_insert_n(array, index, data, 1); +} + +/* Return a mutable pointer to the index-th element, or NULL when invalid. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index) +{ + if (dynamic_array_validate(array) < 0 || index >= array->count) { + if (array != NULL && array->element_size != 0) + errno = EINVAL; + return NULL; + } + return (unsigned char *) array->data + index * array->element_size; +} + +/* Return a read-only pointer to the index-th element, or NULL when invalid. */ +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index) +{ + if (array == NULL || array->element_size == 0) { + errno = EINVAL; + return NULL; + } + if (index >= array->count) { + errno = EINVAL; + return NULL; + } + return (const unsigned char *) array->data + index * array->element_size; +} diff --git a/src/dynamic-array.h b/src/dynamic-array.h new file mode 100644 index 00000000..fb4c7e3c --- /dev/null +++ b/src/dynamic-array.h @@ -0,0 +1,251 @@ +/* + * Generic growable array of trivially-copyable elements. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +typedef struct dynamic_array { + void *data; + size_t count; + size_t capacity; + size_t element_size; +} dynamic_array_t; + +#if defined(__GNUC__) || defined(__clang__) +#define DYNAMIC_ARRAY_INLINE static inline __attribute__((unused)) +#else +#define DYNAMIC_ARRAY_INLINE static inline +#endif + +/* Reset type metadata on a failed first-touch typed operation. */ +static inline int dynamic_array_typed_result(dynamic_array_t *array, + int was_uninitialized, + int result) +{ + if (result < 0 && was_uninitialized && array != NULL && + array->data == NULL && array->count == 0 && array->capacity == 0) + array->element_size = 0; + return result; +} + +/* Set the element size on the first typed operation and report that change. */ +static inline int dynamic_array_typed_prepare(dynamic_array_t *array, + size_t element_size) +{ + int was_uninitialized = array != NULL && array->element_size == 0; + if (was_uninitialized) + array->element_size = element_size; + return was_uninitialized; +} + +/* Initialize an array for elements of element_size bytes without allocation. + * This fresh initializer is safe on an uninitialized automatic object; destroy + * an existing array before reinitializing it. */ +int dynamic_array_init(dynamic_array_t *array, size_t element_size); + +/* Initialize and reserve initial_capacity element slots. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing array before reinitializing it. */ +int dynamic_array_init_with_capacity(dynamic_array_t *array, + size_t element_size, + size_t initial_capacity); + +/* Release storage and restore the all-zero state. */ +void dynamic_array_destroy(dynamic_array_t *array); + +/* Ensure room for extra elements beyond the current count. */ +int dynamic_array_reserve(dynamic_array_t *array, size_t extra); + +/* Set the logical element count. Newly exposed elements are zeroed. */ +int dynamic_array_resize(dynamic_array_t *array, size_t count); + +/* Append one element, or count elements when the three-argument form is + * used. The macro keeps both forms available to callers. */ +int dynamic_array_append_one(dynamic_array_t *array, const void *data); +int dynamic_array_append_n(dynamic_array_t *array, + const void *data, + size_t count); +#define DYNAMIC_ARRAY_APPEND_PICK(_1, _2, _3, NAME, ...) NAME +#define dynamic_array_append(...) \ + DYNAMIC_ARRAY_APPEND_PICK(__VA_ARGS__, dynamic_array_append_n, \ + dynamic_array_append_one, \ + dynamic_array_append_dummy) \ + (__VA_ARGS__) + +/* Insert one element, or count elements in the four-argument form. */ +int dynamic_array_insert_one(dynamic_array_t *array, + size_t index, + const void *data); +int dynamic_array_insert_n(dynamic_array_t *array, + size_t index, + const void *data, + size_t count); +#define DYNAMIC_ARRAY_INSERT_PICK(_1, _2, _3, _4, NAME, ...) NAME +#define dynamic_array_insert(...) \ + DYNAMIC_ARRAY_INSERT_PICK(__VA_ARGS__, dynamic_array_insert_n, \ + dynamic_array_insert_one, \ + dynamic_array_insert_dummy) \ + (__VA_ARGS__) + +/* Return an element pointer, or NULL with errno=EINVAL for a bad index. */ +void *dynamic_array_at(dynamic_array_t *array, size_t index); +const void *dynamic_array_at_const(const dynamic_array_t *array, size_t index); + +/* Generate a small type-safe facade over the raw container. The facade owns + * no additional state; all growth and copying remains in dynamic-array.c. + * A typed object must be zero-initialized before first-touch operations such + * as append, insert, reserve, or resize; the explicit init functions are safe + * on a fresh automatic object and establish the metadata themselves. + */ +#define DYNAMIC_ARRAY_DEFINE(name, type) \ + typedef struct name { \ + dynamic_array_t raw; \ + } name##_t; \ + \ + /* Initialize the typed array with no preallocated storage. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ + DYNAMIC_ARRAY_INLINE int name##_init(name##_t *array) \ + { \ + return dynamic_array_init(array != NULL ? &array->raw : NULL, \ + sizeof(type)); \ + } \ + /* Initialize typed array and preallocate initial slots. This fresh \ + * initializer is safe on an uninitialized automatic object; destroy an \ + * existing array before reinitializing it. */ \ + DYNAMIC_ARRAY_INLINE int name##_init_with_capacity( \ + name##_t *array, size_t initial_capacity) \ + { \ + return dynamic_array_init_with_capacity( \ + array != NULL ? &array->raw : NULL, sizeof(type), \ + initial_capacity); \ + } \ + /* Destroy the typed array and release backing storage. */ \ + DYNAMIC_ARRAY_INLINE void name##_destroy(name##_t *array) \ + { \ + if (array != NULL) \ + dynamic_array_destroy(&array->raw); \ + } \ + /* Reserve extra slots in the typed array. */ \ + DYNAMIC_ARRAY_INLINE int name##_reserve(name##_t *array, size_t extra) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_reserve(array != NULL ? &array->raw : NULL, extra)); \ + } \ + /* Resize typed array, zero-filling newly visible elements. */ \ + DYNAMIC_ARRAY_INLINE int name##_resize(name##_t *array, size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_resize(array != NULL ? &array->raw : NULL, count)); \ + } \ + /* Append one value through a typed pointer. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_ptr(name##_t *array, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, value, \ + 1)); \ + } \ + /* Append one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_value(name##_t *array, type value) \ + { \ + return name##_append_ptr(array, &value); \ + } \ + /* Accept either a value or a pointer to a value while retaining compile- \ + * time checking of the element type. */ \ + DYNAMIC_ARRAY_INLINE int name##_append(name##_t *array, const type *value) \ + { \ + return name##_append_ptr(array, value); \ + } \ + /* Append a typed range of values. */ \ + DYNAMIC_ARRAY_INLINE int name##_append_n(name##_t *array, \ + const type *values, size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_append(array != NULL ? &array->raw : NULL, values, \ + count)); \ + } \ + /* Insert one typed value through a pointer form. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_ptr(name##_t *array, size_t index, \ + const type *value) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + value, 1)); \ + } \ + /* Insert one typed value by value. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_value(name##_t *array, \ + size_t index, type value) \ + { \ + return name##_insert_ptr(array, index, &value); \ + } \ + /* Insert one typed value from a pointer argument. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert(name##_t *array, size_t index, \ + const type *value) \ + { \ + return name##_insert_ptr(array, index, value); \ + } \ + /* Insert a typed range of values at index. */ \ + DYNAMIC_ARRAY_INLINE int name##_insert_n(name##_t *array, size_t index, \ + const type *values, size_t count) \ + { \ + int was_uninitialized = dynamic_array_typed_prepare( \ + array != NULL ? &array->raw : NULL, sizeof(type)); \ + return dynamic_array_typed_result( \ + array != NULL ? &array->raw : NULL, was_uninitialized, \ + dynamic_array_insert(array != NULL ? &array->raw : NULL, index, \ + values, count)); \ + } \ + /* Return a typed pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE type *name##_at(name##_t *array, size_t index) \ + { \ + return (type *) dynamic_array_at(array != NULL ? &array->raw : NULL, \ + index); \ + } \ + /* Return a typed const pointer to the element at index. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_at_const(const name##_t *array, \ + size_t index) \ + { \ + return (const type *) dynamic_array_at_const( \ + array != NULL ? &array->raw : NULL, index); \ + } \ + /* Access the underlying typed data pointer. */ \ + DYNAMIC_ARRAY_INLINE type *name##_data(name##_t *array) \ + { \ + return array != NULL ? (type *) array->raw.data : NULL; \ + } \ + /* Access the underlying typed const data pointer. */ \ + DYNAMIC_ARRAY_INLINE const type *name##_data_const(const name##_t *array) \ + { \ + return array != NULL ? (const type *) array->raw.data : NULL; \ + } \ + /* Query current number of elements in the typed array. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_count(const name##_t *array) \ + { \ + return array != NULL ? array->raw.count : 0; \ + } \ + /* Query current allocated capacity. */ \ + DYNAMIC_ARRAY_INLINE size_t name##_capacity(const name##_t *array) \ + { \ + return array != NULL ? array->raw.capacity : 0; \ + } diff --git a/src/runtime/fork-state.c b/src/runtime/fork-state.c index 43b75d49..9ff5a799 100644 --- a/src/runtime/fork-state.c +++ b/src/runtime/fork-state.c @@ -926,6 +926,13 @@ int fork_ipc_recv_process_state(int ipc_fd, guest_t *g, signal_state_t *sig) return -1; g->nregions = (int) recv_regions; + /* Every VMA present in the serialized parent snapshot is inherited by + * this child, regardless of whether the parent itself created it after an + * earlier fork. New mappings added in this process start unmarked through + * guest_region_add_ex_owned[_gpa]. + */ + for (int i = 0; i < g->nregions; i++) + g->regions[i].inherited_at_fork = true; g->regions_tracker_stale = (regions_tracker_stale != 0) || (num_guest_regions > recv_regions); diff --git a/src/runtime/fork-state.h b/src/runtime/fork-state.h index d4a5663e..b6f8331e 100644 --- a/src/runtime/fork-state.h +++ b/src/runtime/fork-state.h @@ -19,7 +19,7 @@ /* Fork IPC protocol identity. Bump this whenever the header layout or ordered * fork payload changes incompatibly. */ -#define FORK_IPC_PROTOCOL_MAGIC 0x454C464EU /* "ELFN" */ +#define FORK_IPC_PROTOCOL_MAGIC 0x454C464FU /* "ELFO" */ #define IPC_MAGIC_HEADER FORK_IPC_PROTOCOL_MAGIC #define IPC_MAGIC_SENTINEL 0x454C4F4BU /* "ELOK" */ diff --git a/src/runtime/procemu.c b/src/runtime/procemu.c index ed83fcc7..97911b31 100644 --- a/src/runtime/procemu.c +++ b/src/runtime/procemu.c @@ -10,10 +10,17 @@ * (caller falls through to real syscall). */ -/* Maximum /proc/self/maps entries. Array is sized to this; loop bounds use - * MAPS_ENTRY_MAX - 1 to leave room for safe increment. +/* Initial capacity for the transient /proc/self/maps and /proc/self/smaps VMA + * snapshot. The region tracker documents that coalescing leaves typical + * workloads at roughly 50 tracked regions (see core/guest.h), so 64 avoids an + * immediate growth in that case. The array grows as needed while mmap_lock is + * held; keep a hard ceiling tied to the guest's region tables so maps/smaps + * can enumerate every tracked VMA while keeping transient snapshot memory + * bounded. */ -#define MAPS_ENTRY_MAX 256 +#define MAPS_ENTRY_INITIAL_CAP 64 +#define MAPS_ENTRY_MAX \ + (GUEST_MAX_REGIONS + GUEST_MAX_PREANNOUNCED * (GUEST_MAX_REGIONS + 1)) /* Bound the transient host-PID snapshot used by /proc/net enumeration. This * is an output-work limit, not the dynamically growing lifecycle-table cap. @@ -51,6 +58,7 @@ #include #include +#include "string-builder.h" #include "utils.h" #include "debug/log.h" @@ -94,58 +102,180 @@ typedef struct { uint64_t start, end; int prot, flags; uint64_t offset; + bool inherited_at_fork; char name[64]; + /* Preserve the producer order for equal-start entries when qsort() is + * used below. Existing snapshots placed equal-start entries after one + * another in append order, and keeping that order avoids changing the + * handling of malformed/overlapping shadow metadata. */ + size_t order; } maps_entry_t; -static void maps_entry_insert(maps_entry_t *entries, - int *nentries, - uint64_t start, - uint64_t end, - int prot, - int flags, - uint64_t offset, - const char *name) +/* A growable VMA snapshot. The generated facade keeps element-size and + * allocation bookkeeping in the generic array implementation. */ +DYNAMIC_ARRAY_DEFINE(maps_entries, maps_entry_t) + +/* Round a VMA endpoint up to the 4 KiB granularity exposed by procfs without + * allowing the addition to wrap. There is no representable page-aligned + * endpoint above UINT64_MAX - 0xFFF, so saturate to UINT64_MAX and let the + * caller's normal end <= start validation handle an empty interval. + */ +static uint64_t maps_align_up_page(uint64_t value) { - if (*nentries >= MAPS_ENTRY_MAX || end <= start) - return; + const uint64_t mask = 0xFFFULL; + if (value > UINT64_MAX - mask) + return UINT64_MAX; + return (value + mask) & ~mask; +} - int i = *nentries; - while (i > 0 && entries[i - 1].start > start) { - entries[i] = entries[i - 1]; - i--; +/* Translate a shadow VMA's cursor into its file offset. A wrapped offset would + * produce a syntactically valid but semantically wrong maps entry, so surface + * the overflow to the intercepted open instead. + */ +static int maps_shadow_offset(const guest_region_t *shadow, + uint64_t shadow_start, + uint64_t cursor, + uint64_t *offset_out) +{ + uint64_t delta = cursor - shadow_start; + if (delta > UINT64_MAX - shadow->offset) { + errno = EOVERFLOW; + return -1; + } + *offset_out = shadow->offset + delta; + return 0; +} + +static int maps_entries_append_entry(maps_entries_t *entries, + uint64_t start, + uint64_t end, + int prot, + int flags, + uint64_t offset, + const char *name, + bool inherited_at_fork) +{ + if (end <= start) + return 0; + if (maps_entries_count(entries) >= MAPS_ENTRY_MAX) { + errno = ENOMEM; + return -1; } + if (maps_entries_count(entries) == 0 && + maps_entries_reserve(entries, MAPS_ENTRY_INITIAL_CAP) < 0) + return -1; - maps_entry_t *e = &entries[i]; - e->start = start; - e->end = end; - e->prot = prot; - e->flags = flags; - e->offset = offset; + maps_entry_t value = { + .start = start, + .end = end, + .prot = prot, + .flags = flags, + .offset = offset, + .inherited_at_fork = inherited_at_fork, + .order = maps_entries_count(entries), + }; if (name && name[0]) - str_copy_trunc(e->name, name, sizeof(e->name)); + str_copy_trunc(value.name, name, sizeof(value.name)); else - e->name[0] = '\0'; - (*nentries)++; + value.name[0] = '\0'; + + return maps_entries_append_value(entries, value); } -static void maps_entries_merge_adjacent(maps_entry_t *entries, int *nentries) +/* The live-region and shadow-gap producers are each ordered, but their + * outputs interleave. Append both streams while holding mmap_lock and sort + * once after all gaps have been generated. This avoids shifting an already + * populated array for every split shadow gap (the old insertion path was + * quadratic for fragmented snapshots). */ +static int maps_entries_compare_start(const void *lhs, const void *rhs) { - if (*nentries <= 1) + const maps_entry_t *a = lhs; + const maps_entry_t *b = rhs; + if (a->start < b->start) + return -1; + if (a->start > b->start) + return 1; + if (a->order < b->order) + return -1; + if (a->order > b->order) + return 1; + return 0; +} + +static void maps_entries_merge_adjacent(maps_entries_t *entries) +{ + size_t count = maps_entries_count(entries); + if (count <= 1) return; - int out = 0; - for (int i = 1; i < *nentries; i++) { - if (entries[i].start == entries[out].end && - entries[i].prot == entries[out].prot && - entries[i].flags == entries[out].flags && - entries[i].offset == entries[out].offset && - strcmp(entries[i].name, entries[out].name) == 0) { - entries[out].end = entries[i].end; + size_t out = 0; + for (size_t i = 1; i < count; i++) { + maps_entry_t *current = maps_entries_at(entries, i); + maps_entry_t *previous = maps_entries_at(entries, out); + if (current->start == previous->end && + current->prot == previous->prot && + current->flags == previous->flags && + current->offset == previous->offset && + current->inherited_at_fork == previous->inherited_at_fork && + strcmp(current->name, previous->name) == 0) { + previous->end = current->end; + continue; + } + ++out; + if (out != i) + *maps_entries_at(entries, out) = *current; + } + (void) maps_entries_resize(entries, out + 1); +} + +/* Add only the portions of a preannounced interval not covered by live VMAs. + * A shadow VMA must never overlap a realized VMA: strict smaps consumers treat + * overlapping headers as a malformed snapshot. Both inputs are page-rounded + * because that is the granularity exposed by /proc/self/maps. */ +static int maps_entries_append_shadow_gaps(maps_entries_t *entries, + const guest_region_t *shadow, + const guest_region_t *live_regions, + int nlive) +{ + uint64_t shadow_start = shadow->start & ~0xFFFULL; + uint64_t shadow_end = maps_align_up_page(shadow->end); + if (shadow_end <= shadow_start) + return 0; + + uint64_t cursor = shadow_start; + for (int i = 0; i < nlive && cursor < shadow_end; i++) { + uint64_t live_start = live_regions[i].start & ~0xFFFULL; + uint64_t live_end = maps_align_up_page(live_regions[i].end); + if (live_end <= live_start || live_end <= cursor) continue; + if (live_start >= shadow_end) + break; + + if (live_start > cursor) { + uint64_t gap_end = + live_start < shadow_end ? live_start : shadow_end; + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; + if (maps_entries_append_entry( + entries, cursor, gap_end, shadow->prot, shadow->flags, + offset, shadow->name, shadow->inherited_at_fork) < 0) + return -1; } - entries[++out] = entries[i]; + if (live_end > cursor) + cursor = live_end; + } + + if (cursor < shadow_end) { + uint64_t offset; + if (maps_shadow_offset(shadow, shadow_start, cursor, &offset) < 0) + return -1; + if (maps_entries_append_entry(entries, cursor, shadow_end, shadow->prot, + shadow->flags, offset, shadow->name, + shadow->inherited_at_fork) < 0) + return -1; } - *nentries = out + 1; + return 0; } /* Synthetic /sys/devices/system/cpu directory backing store. Populated lazily @@ -445,7 +575,8 @@ static void proc_tmpdir_cleanup(void) /* Remove known files inside // and / */ char path[256]; - const char *files[] = {"stat", "status", "cmdline", "maps", "exe", NULL}; + const char *files[] = {"stat", "status", "cmdline", "maps", + "smaps", "exe", NULL}; char piddir[160]; /* Reconstruct pid subdir by scanning for the first numeric entry */ @@ -1234,6 +1365,7 @@ static const char *ensure_proc_tmpdir(const guest_t *g) populate_proc_snapshot(g, piddir, "status", "/proc/self/status"); populate_proc_snapshot(g, piddir, "cmdline", "/proc/self/cmdline"); populate_proc_snapshot(g, piddir, "maps", "/proc/self/maps"); + populate_proc_snapshot(g, piddir, "smaps", "/proc/self/smaps"); /* Create task subdirectory for /proc/self/task enumeration */ char taskdir[128]; @@ -2330,144 +2462,295 @@ static int pty_open_master(int linux_flags) return master; } -/* Emit /proc/self/maps into a synthetic fd. Merges contiguous regions[] runs - * that came from one mmap, then folds in preannounced[] shadow entries whose - * advertised interval is not yet fully covered by live regions. +/* Build the VMA list shared by /proc/self/maps and /proc/self/smaps. Merges + * contiguous regions[] runs that came from one mmap, then folds in the + * uncovered pieces of preannounced[] shadow entries around live coverage. + * Producers append entries while the lock is held; one sort/merge pass puts + * the interleaved live and shadow streams back into VMA order. * - * Returns a host fd, or -1 on error. Split out of proc_intercept_open to keep - * that dispatcher readable. + * The region and preannounced arrays are mutable from guest mmap/mprotect/ + * munmap operations. Snapshot and merge while mmap_lock is held, then release + * it before formatting output so readers never observe a torn VMA or metadata + * record and output generation does not block page-table mutations. */ -static int proc_open_self_maps(const guest_t *g) +static int proc_build_maps_entries(const guest_t *g, + maps_entries_t *entries_out) { - /* Heap-allocated: guest threads run on host pthreads whose stacks do not - * comfortably hold a 16KiB frame. - */ - const size_t bufsz = 16384; - char *buf = malloc(bufsz); - if (!buf) - return -1; - int off = 0; - - /* Build a flat array of (va_start, va_end, prot, flags, offset, name) from - * regions[] plus /proc/self/maps-only preannounced[] entries. - * preannounced[] is intentionally NOT consulted by mmap conflict detection, - * so advertise-only Rosetta/JIT regions do not trip MAP_FIXED_NOREPLACE - * with -EEXIST. - * - * entries is heap-allocated: MAPS_ENTRY_MAX * sizeof(maps_entry_t) is - * ~24KiB, too large for a guest-thread pthread stack. - */ - maps_entry_t *entries = calloc(MAPS_ENTRY_MAX, sizeof(*entries)); - if (!entries) { - free(buf); + if (!g || !entries_out) { + errno = EINVAL; return -1; } - int nentries = 0; + + maps_entries_t entries = {0}; + int result = -1; + int saved_errno = 0; + + pthread_mutex_lock(&mmap_lock); /* Convert regions[] to maps entries. regions[] is already sorted by start - * address; merge contiguous runs that came from one mmap. + * address. The MAP_SHARED/MAP_ANONYMOUS/MAP_NORESERVE bits are preserved + * in r->flags, which is the single source of truth for the proc snapshot. */ - for (int i = 0; i < g->nregions && nentries < MAPS_ENTRY_MAX; i++) { + int nregions = g->nregions; + if (nregions < 0) + nregions = 0; + if (nregions > GUEST_MAX_REGIONS) + nregions = GUEST_MAX_REGIONS; + for (int i = 0; i < nregions; i++) { const guest_region_t *r = &g->regions[i]; uint64_t start = r->start & ~0xFFFULL; - uint64_t end = (r->end + 0xFFF) & ~0xFFFULL; - - if (nentries > 0 && entries[nentries - 1].end == start && - entries[nentries - 1].prot == r->prot && - entries[nentries - 1].flags == r->flags && - entries[nentries - 1].offset == r->offset && - !strcmp(entries[nentries - 1].name, r->name)) { - entries[nentries - 1].end = end; + uint64_t end = maps_align_up_page(r->end); + size_t count = maps_entries_count(&entries); + maps_entry_t *last = + count > 0 ? maps_entries_at(&entries, count - 1) : NULL; + if (last != NULL && last->end == start && last->prot == r->prot && + last->flags == r->flags && last->offset == r->offset && + last->inherited_at_fork == r->inherited_at_fork && + !strcmp(last->name, r->name)) { + last->end = end; continue; } - maps_entry_insert(entries, &nentries, start, end, r->prot, r->flags, - r->offset, r->name); - } - - /* Add preannounced entries only while they still have an uncovered tail. - * Once the union of live regions covers the full advertised interval, - * suppress the shadow entry so /proc/self/maps shows only the realized - * split VMAs. A partial union must stay visible because some - * reserved-but-not-realized span remains to advertise. - */ - for (int i = 0; i < g->npreannounced && nentries < MAPS_ENTRY_MAX; i++) { + if (maps_entries_append_entry(&entries, start, end, r->prot, r->flags, + r->offset, r->name, + r->inherited_at_fork) < 0) + goto out_unlock; + } + + /* Add only uncovered portions of each preannounced interval. Keeping the + * shadow VMA whole when a live mapping realizes its middle produces + * overlapping maps/smaps headers; subtract every covered live interval + * instead, preserving any reserved-but-not-realized gaps. */ + int npreannounced = g->npreannounced; + if (npreannounced < 0) + npreannounced = 0; + if (npreannounced > GUEST_MAX_PREANNOUNCED) + npreannounced = GUEST_MAX_PREANNOUNCED; + for (int i = 0; i < npreannounced; i++) { const guest_region_t *r = &g->preannounced[i]; - bool shadowed = false; - uint64_t covered_end = r->start; - - for (int j = 0; j < g->nregions; j++) { - const guest_region_t *live = &g->regions[j]; - - if (live->end <= covered_end) - continue; - if (live->start > covered_end) - break; + if (maps_entries_append_shadow_gaps(&entries, r, g->regions, nregions) < + 0) + goto out_unlock; + } + if (maps_entries_count(&entries) > 1) + qsort(maps_entries_data(&entries), maps_entries_count(&entries), + sizeof(maps_entry_t), maps_entries_compare_start); + maps_entries_merge_adjacent(&entries); + result = (int) maps_entries_count(&entries); + +out_unlock: + saved_errno = errno; + pthread_mutex_unlock(&mmap_lock); + if (result < 0) { + maps_entries_destroy(&entries); + errno = saved_errno; + return -1; + } + *entries_out = entries; + return (int) maps_entries_count(&entries); +} - covered_end = live->end; - if (covered_end >= r->end) { - shadowed = true; - break; - } +/* Format the common VMA header. The maps and smaps header must stay byte-for- + * byte compatible so consumers can use either file interchangeably. + */ +static int proc_format_maps_header(const maps_entry_t *e, + char *header, + size_t headersz) +{ + char perms[5]; + perms[0] = (e->prot & LINUX_PROT_READ) ? 'r' : '-'; + perms[1] = (e->prot & LINUX_PROT_WRITE) ? 'w' : '-'; + perms[2] = (e->prot & LINUX_PROT_EXEC) ? 'x' : '-'; + perms[3] = (e->flags & LINUX_MAP_SHARED) ? 's' : 'p'; + perms[4] = '\0'; + + int header_len = + snprintf(header, headersz, "%llx-%llx %s %08llx 00:00 0", + (unsigned long long) e->start, (unsigned long long) e->end, + perms, (unsigned long long) e->offset); + if (header_len < 0) + return -1; + if ((size_t) header_len >= headersz) + header_len = (int) headersz - 1; + + if (e->name[0]) { + while (header_len < MAPS_NAME_COLUMN && + (size_t) header_len < headersz - 1) + header[header_len++] = ' '; + int n = snprintf(header + header_len, headersz - (size_t) header_len, + "%s", e->name); + if (n > 0) { + if ((size_t) n >= headersz - (size_t) header_len) + n = (int) (headersz - (size_t) header_len - 1); + header_len += n; } + } else if ((size_t) header_len < headersz - 1) { + header[header_len++] = ' '; + } + header[header_len] = '\0'; + return header_len; +} - if (shadowed) - continue; +/* Release the resources shared by maps/smaps output without hiding the errno + * from the operation that produced result. + */ +static int proc_finish_maps_output(int result, + maps_entries_t *entries, + string_builder_t *builder) +{ + int saved_errno = errno; + maps_entries_destroy(entries); + string_builder_destroy(builder); + errno = saved_errno; + return result; +} - maps_entry_insert(entries, &nentries, r->start & ~0xFFFULL, - (r->end + 0xFFFULL) & ~0xFFFULL, r->prot, r->flags, - r->offset, r->name); - } - maps_entries_merge_adjacent(entries, &nentries); +/* Emit /proc/self/maps into a synthetic fd. Addresses are page-aligned and + * output matches the Linux maps header format. + */ +static int proc_open_self_maps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 256; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; /* Emit lines after merging so buffer accounting is centralized. */ - for (int i = 0; i < nentries && off < (int) bufsz - 256; i++) { - const maps_entry_t *e = &entries[i]; - char perms[5]; - perms[0] = (e->prot & 0x1) ? 'r' : '-'; - perms[1] = (e->prot & 0x2) ? 'w' : '-'; - perms[2] = (e->prot & 0x4) ? 'x' : '-'; - perms[3] = (e->flags & 0x01) ? 's' : 'p'; - perms[4] = '\0'; - - /* Format matches real Linux /proc//maps exactly: - * %lx-%lx %s %08lx %02x:%02x %lu %s\n - * Verified against strace in a real Lima VZ VM. - */ + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); char line[256]; - int lineoff = - snprintf(line, sizeof(line), "%llx-%llx %s %08llx 00:00 0", - (unsigned long long) e->start, (unsigned long long) e->end, - perms, (unsigned long long) e->offset); - /* Cap lineoff to buffer size (snprintf may return more than available - * on truncation) + int line_len = proc_format_maps_header(e, line, sizeof(line)); + if (line_len < 0 || + string_builder_appendf(&builder, "%.*s\n", line_len, line) < 0) + goto out; + } + + log_debug("/proc/self/maps (%zu bytes):\n%.*s", + string_builder_length(&builder), + (int) string_builder_length(&builder), + string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : ""); + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); + +out: + return proc_finish_maps_output(result, &entries, &builder); +} + +/* Emit a Linux-shaped /proc/self/smaps approximation. The runtime tracks + * guest VMAs and logical fork snapshots, but it cannot observe kernel page + * residency or dirty bits on the host. Writable private anonymous VMAs that + * were present in the most recent fork snapshot report their full VMA size as + * Shared_Dirty; newly-created VMAs are excluded. All other fields that + * require kernel page accounting are stable zeroes. + */ +static int proc_open_self_smaps(const guest_t *g) +{ + maps_entries_t entries = {0}; + int nentries = proc_build_maps_entries(g, &entries); + if (nentries < 0) + return -1; + + string_builder_t builder = {0}; + size_t initial_capacity = (size_t) nentries * 768; + int result = -1; + if (string_builder_init(&builder, initial_capacity) < 0) + goto out; + + for (int i = 0; i < nentries; i++) { + const maps_entry_t *e = maps_entries_at_const(&entries, (size_t) i); + char header[256]; + int header_len = proc_format_maps_header(e, header, sizeof(header)); + if (header_len < 0) + goto out; + + uint64_t size_kb = (e->end - e->start) / 1024; + bool anonymous = (e->flags & LINUX_MAP_ANONYMOUS) != 0; + bool shared = (e->flags & LINUX_MAP_SHARED) != 0; + bool noreserve = (e->flags & LINUX_MAP_NORESERVE) != 0; + bool private_anon = + (e->flags & LINUX_MAP_PRIVATE) && anonymous && !shared; + bool logical_shared_dirty = e->inherited_at_fork && private_anon && + (e->prot & LINUX_PROT_WRITE); + uint64_t shared_dirty_kb = logical_shared_dirty ? size_kb : 0; + + char vmflags[64]; + size_t vmflags_len = 0; +#define APPEND_VMFLAG(flag) \ + do { \ + const char *token = (flag); \ + size_t token_len = strlen(token); \ + if (vmflags_len + token_len + 1 < sizeof(vmflags)) { \ + vmflags[vmflags_len++] = ' '; \ + memcpy(vmflags + vmflags_len, token, token_len); \ + vmflags_len += token_len; \ + } \ + } while (0) + /* Only flags directly evidenced by the tracked VMA are reported. In + * particular, do not invent Linux max-permission/accounting flags + * (mr/mw/me/ac/sd/etc.) that the emulator cannot observe. */ - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - if (e->name[0]) { - while (lineoff < MAPS_NAME_COLUMN && - lineoff < (int) sizeof(line) - 1) - line[lineoff++] = ' '; - int n = - snprintf(line + lineoff, sizeof(line) - lineoff, "%s", e->name); - if (n > 0) - lineoff += n; - if (lineoff >= (int) sizeof(line)) - lineoff = (int) sizeof(line) - 1; - } else if (lineoff < (int) sizeof(line) - 1) { - line[lineoff++] = ' '; - } - int wrote = snprintf(buf + off, bufsz - off, "%.*s\n", lineoff, line); - if (wrote > 0 && off + wrote < (int) bufsz) - off += wrote; - else - break; /* Stop before truncating a maps line. */ - } + if (e->prot & LINUX_PROT_READ) + APPEND_VMFLAG("rd"); + if (e->prot & LINUX_PROT_WRITE) + APPEND_VMFLAG("wr"); + if (e->prot & LINUX_PROT_EXEC) + APPEND_VMFLAG("ex"); + if (shared) + APPEND_VMFLAG("sh"); + if (noreserve) + APPEND_VMFLAG("nr"); +#undef APPEND_VMFLAG + vmflags[vmflags_len] = '\0'; + + if (string_builder_appendf( + &builder, + "%.*s\n" + "Size: %llu kB\n" + "KernelPageSize: 4 kB\n" + "MMUPageSize: 4 kB\n" + "Rss: 0 kB\n" + "Pss: 0 kB\n" + "Pss_Dirty: 0 kB\n" + "Shared_Clean: 0 kB\n" + "Shared_Dirty: %llu kB\n" + "Private_Clean: 0 kB\n" + "Private_Dirty: 0 kB\n" + "Referenced: 0 kB\n" + "Anonymous: 0 kB\n" + "KSM: 0 kB\n" + "LazyFree: 0 kB\n" + "AnonHugePages: 0 kB\n" + "ShmemPmdMapped: 0 kB\n" + "FilePmdMapped: 0 kB\n" + "Shared_Hugetlb: 0 kB\n" + "Private_Hugetlb: 0 kB\n" + "Swap: 0 kB\n" + "SwapPss: 0 kB\n" + "Locked: 0 kB\n" + "THPeligible: 0\n" + "ProtectionKey: 0\n" + "VmFlags:%s\n", + header_len, header, (unsigned long long) size_kb, + (unsigned long long) shared_dirty_kb, vmflags) < 0) + goto out; + } + + result = proc_synthetic_fd(string_builder_data_const(&builder) + ? string_builder_data_const(&builder) + : "", + string_builder_length(&builder)); - log_debug("/proc/self/maps (%d bytes):\n%.*s", off, off, buf); - int fd = proc_synthetic_fd(buf, off); - free(entries); - free(buf); - return fd; +out: + return proc_finish_maps_output(result, &entries, &builder); } /* Emit /proc/meminfo from host sysctl (HW_MEMSIZE) plus mach vm_statistics64, @@ -3076,6 +3359,12 @@ int proc_intercept_open(const guest_t *g, if (!strcmp(path, "/proc/self/maps")) return proc_open_self_maps(g); + /* /proc/self/smaps -> Linux-shaped VMA blocks with tracked VMA metadata + * and the coarse fork Shared_Dirty compatibility signal. + */ + if (!strcmp(path, "/proc/self/smaps")) + return proc_open_self_smaps(g); + /* /proc/uptime -> synthetic uptime in seconds. Uses sysctl(KERN_BOOTTIME), * same as sys_sysinfo() in syscall/sys.c. Idle time is 0 (no meaningful * macOS equivalent). @@ -3680,6 +3969,7 @@ int proc_intercept_stat(const char *path, struct stat *st) "/proc/self/status", "/proc/self/cmdline", "/proc/self/maps", + "/proc/self/smaps", "/proc/self/exe", "/proc/self/environ", "/proc/self/auxv", diff --git a/src/runtime/procemu.h b/src/runtime/procemu.h index 32d8767a..3e66ad0e 100644 --- a/src/runtime/procemu.h +++ b/src/runtime/procemu.h @@ -24,7 +24,7 @@ #define PROC_NOT_INTERCEPTED (-2) /* Intercept openat for /proc and /dev paths. The guest_t pointer is needed to - * generate /proc/self/maps from region data. + * generate /proc/self/maps and /proc/self/smaps from region data. * Returns a host fd on match (caller should fd_alloc it), -1 on error with * errno set, or PROC_NOT_INTERCEPTED if the path is not intercepted. */ diff --git a/src/string-builder.c b/src/string-builder.c new file mode 100644 index 00000000..97d7bec9 --- /dev/null +++ b/src/string-builder.c @@ -0,0 +1,244 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "string-builder.h" + +#include +#include +#include +#include +#include +#include + +/* Set EILSEQ for invalid input and report failure. */ +static int string_builder_invalid(void) +{ + errno = EILSEQ; + return -1; +} + +/* Initialize storage and establish an empty, NUL-terminated builder. This + * fresh initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. + */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity) +{ + if (builder == NULL) + return string_builder_invalid(); + builder->storage = (string_builder_storage_t) {0}; + if (initial_capacity == 0) { + return 0; + } + if (string_builder_storage_init_with_capacity(&builder->storage, + initial_capacity) < 0) + return -1; + string_builder_data(builder)[0] = '\0'; + return 0; +} + +/* Release storage and reset the logical length. */ +void string_builder_destroy(string_builder_t *builder) +{ + if (builder == NULL) + return; + string_builder_storage_destroy(&builder->storage); +} + +/* Reserve enough capacity for extra bytes after the current contents. */ +int string_builder_reserve(string_builder_t *builder, size_t extra) +{ + if (builder == NULL) + return string_builder_invalid(); + if (extra == 0 && string_builder_storage_count(&builder->storage) == 0 && + string_builder_capacity(builder) == 0) + return 0; + + /* The dynamic array counts payload elements. Reserve one additional char + * for the string builder's trailing NUL. */ + if (extra == SIZE_MAX) { + errno = EOVERFLOW; + return -1; + } + if (string_builder_storage_reserve(&builder->storage, extra + 1) < 0) + return -1; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; + return 0; +} + +/* Locate a source span that aliases the builder allocation. The offset must be + * captured before reserve because reserve may move the allocation. */ +static int string_builder_source_offset(const string_builder_t *builder, + const void *source, + size_t length, + size_t *offset) +{ + const char *base_ptr = string_builder_data_const(builder); + size_t capacity = string_builder_capacity(builder); + if (base_ptr == NULL || source == NULL) + return 0; + uintptr_t base = (uintptr_t) (const void *) base_ptr; + uintptr_t address = (uintptr_t) source; + if (address < base) + return 0; + uintptr_t delta = address - base; + if (delta > (uintptr_t) SIZE_MAX) + return 0; + size_t start = (size_t) delta; + if (start > capacity || length > capacity - start) + return 0; + *offset = start; + return 1; +} + +/* Commit string bytes through the generic array and restore the terminator. */ +static int string_builder_commit_append(string_builder_t *builder, + const char *data, + size_t len) +{ + if (string_builder_storage_append_n(&builder->storage, data, len) < 0) + return -1; + string_builder_data( + builder)[string_builder_storage_count(&builder->storage)] = '\0'; + return 0; +} + +/* Append a C string and keep the builder NUL-terminated. */ +int string_builder_append(string_builder_t *builder, const char *text) +{ + if (builder == NULL || text == NULL) + return string_builder_invalid(); + + size_t len = strlen(text); + if (len == 0) + return 0; + + size_t source_offset = 0; + int aliases = + string_builder_source_offset(builder, text, len, &source_offset); + if (string_builder_reserve(builder, len) < 0) + return -1; + if (aliases) + text = string_builder_data_const(builder) + source_offset; + return string_builder_commit_append(builder, text, len); +} + +/* Convert a formatting failure into the module's documented errno values. */ +static int string_builder_format_failure(void) +{ + if (errno != EOVERFLOW && errno != EILSEQ) + errno = EILSEQ; + return -1; +} + +/* Format into separate storage before touching the builder. Besides avoiding + * writes through an aliased format string, this keeps %s arguments that point + * into the builder valid even when appending the result grows the allocation. + */ +int string_builder_appendf(string_builder_t *builder, const char *format, ...) +{ + int saved_errno; + int formatted_len; + size_t visible_len; + char *formatted = NULL; + va_list arguments; + va_list sizing; + va_list rendering; + char sizing_sink; + + if (builder == NULL || format == NULL) + return string_builder_invalid(); + + saved_errno = errno; + va_start(arguments, format); + + va_copy(sizing, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + /* A non-NULL destination keeps static analyzers from treating the + * standards-sanctioned n == 0 sizing call as a null dereference. The + * destination is never written when its size is zero. */ + formatted_len = vsnprintf(&sizing_sink, 0, format, sizing); +#pragma clang diagnostic pop + va_end(sizing); + if (formatted_len < 0) { + string_builder_format_failure(); + goto out; + } + + formatted = malloc((size_t) formatted_len + 1); + if (formatted == NULL) { + errno = ENOMEM; + goto out; + } + + va_copy(rendering, arguments); + errno = 0; +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + int rendered_len = + vsnprintf(formatted, (size_t) formatted_len + 1, format, rendering); +#pragma clang diagnostic pop + va_end(rendering); + if (rendered_len < 0 || rendered_len != formatted_len) { + string_builder_format_failure(); + goto out; + } + + /* Preserve the builder's C-string semantics for formatted NUL bytes. */ + visible_len = strlen(formatted); + if (visible_len == 0) { + errno = saved_errno; + free(formatted); + va_end(arguments); + return 0; + } + if (string_builder_reserve(builder, visible_len) < 0) + goto out; + if (string_builder_commit_append(builder, formatted, visible_len) < 0) + goto out; + + errno = saved_errno; + free(formatted); + va_end(arguments); + return 0; + +out: + free(formatted); + va_end(arguments); + return -1; +} + +/* Return mutable storage for the builder, if it has been allocated. */ +char *string_builder_data(string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_data(&builder->storage) + : NULL; +} + +/* Return const storage for the builder, if it has been allocated. */ +const char *string_builder_data_const(const string_builder_t *builder) +{ + return builder != NULL + ? string_builder_storage_data_const(&builder->storage) + : NULL; +} + +/* Return the number of data bytes currently stored. */ +size_t string_builder_length(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_count(&builder->storage) + : 0; +} + +/* Return allocated capacity in bytes, including the terminating NUL. */ +size_t string_builder_capacity(const string_builder_t *builder) +{ + return builder != NULL ? string_builder_storage_capacity(&builder->storage) + : 0; +} diff --git a/src/string-builder.h b/src/string-builder.h new file mode 100644 index 00000000..c0942071 --- /dev/null +++ b/src/string-builder.h @@ -0,0 +1,65 @@ +/* + * Growable, NUL-terminated C-string builder. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "dynamic-array.h" + +DYNAMIC_ARRAY_DEFINE(string_builder_storage, char) + +/* The storage representation is an implementation detail; callers should use + * the accessors below. The generated count is the string length; capacity is + * measured in bytes and includes the trailing NUL slot. + */ +typedef struct string_builder { + string_builder_storage_t storage; +} string_builder_t; + +/* Initialize a builder, reserving initial_capacity bytes including the NUL. + * A zero capacity leaves storage unallocated for lazy growth. This fresh + * initializer is safe on an uninitialized automatic object; destroy an + * existing builder before reinitializing it. Returns 0 on success or -1 with + * errno set on invalid input or allocation failure. + */ +int string_builder_init(string_builder_t *builder, size_t initial_capacity); + +/* Release the builder storage and reset its length. Safe to call with NULL. */ +void string_builder_destroy(string_builder_t *builder); + +/* Ensure room for extra string bytes plus the terminating NUL. */ +int string_builder_reserve(string_builder_t *builder, size_t extra); + +/* Append a C string, accepting aliases into the builder's storage. As with + * standard C string functions, the first NUL terminates the input. A NULL + * pointer is invalid. The resulting C string is always NUL-terminated. + */ +int string_builder_append(string_builder_t *builder, const char *text); + +/* Append formatted text using printf-style arguments. A formatted NUL byte + * terminates the appended C string prefix. + */ +#if defined(__GNUC__) || defined(__clang__) +__attribute__((format(printf, 2, 3))) +#endif +int string_builder_appendf(string_builder_t *builder, const char *format, ...); + +/* Return mutable builder data, or NULL when builder is NULL or unallocated. + * Callers must preserve the no-embedded-NUL C-string invariant. + */ +char *string_builder_data(string_builder_t *builder); + +/* Return read-only builder data, or NULL when builder is NULL or unallocated. + */ +const char *string_builder_data_const(const string_builder_t *builder); + +/* Return the number of data bytes currently stored, excluding the NUL. */ +size_t string_builder_length(const string_builder_t *builder); + +/* Return allocated capacity in bytes, including space for the NUL. */ +size_t string_builder_capacity(const string_builder_t *builder); diff --git a/src/syscall/mem.c b/src/syscall/mem.c index a5477c5e..7ea6b2ef 100644 --- a/src/syscall/mem.c +++ b/src/syscall/mem.c @@ -75,6 +75,7 @@ typedef struct { uint64_t start; uint64_t end; uint64_t gpa_base; + uint64_t vma_id; int prot; int flags; uint64_t offset; @@ -83,6 +84,7 @@ typedef struct { uint64_t overlay_start; uint64_t overlay_end; bool backing_ro; + bool inherited_at_fork; char name[sizeof(((guest_region_t *) 0)->name)]; } region_snapshot_t; @@ -207,19 +209,236 @@ static void mark_overlay_metadata_range(guest_t *g, } } -/* Mark the region spanning exactly [start, end) as backed by a fd that lost - * write access, so sys_mprotect rejects a later PROT_WRITE upgrade. Exact match - * (not overlap) because callers use this right after installing a single - * freshly-added region. +/* Mark every region overlapping [start, end) as backed by a fd that lost write + * access, so sys_mprotect rejects a later PROT_WRITE upgrade. mremap can split + * an inherited VMA at the fork boundary, so callers must not require one + * exact region match here. */ static void mark_region_backing_ro(guest_t *g, uint64_t start, uint64_t end) { for (int i = 0; i < g->nregions; i++) { - if (g->regions[i].start == start && g->regions[i].end == end) { - g->regions[i].backing_ro = true; + if (g->regions[i].start >= end) break; + if (g->regions[i].end <= start) + continue; + g->regions[i].backing_ro = true; + } +} + +/* Track an mremap result without losing the fork boundary inside an in-place + * growth or a moved mapping. Bytes copied from the old VMA were present at the + * fork snapshot; an extension is new child-private address space and must + * remain unmarked. Keep the two portions as separate VMAs even when all other + * metadata matches so the synthetic smaps view can distinguish them. + */ +static bool mremap_backings_match(const guest_region_t *a, + const guest_region_t *b) +{ + if (a->backing_fd < 0 || b->backing_fd < 0) + return a->backing_fd < 0 && b->backing_fd < 0; + if (a->backing_fd == b->backing_fd) + return true; + + struct stat sa, sb; + if (fstat(a->backing_fd, &sa) < 0 || fstat(b->backing_fd, &sb) < 0) + return false; + return sa.st_dev == sb.st_dev && sa.st_ino == sb.st_ino; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t gpa_base; + uint64_t offset; + int backing_fd; /* borrowed from the region tracker */ + bool overlay_active; + uint64_t overlay_start; + uint64_t overlay_end; +} mremap_source_segment_t; + +typedef struct { + const guest_region_t *first; + uint64_t inherited_prefix; + int nsegments; + mremap_source_segment_t *segments; +} mremap_source_t; + +static void dispose_mremap_source(mremap_source_t *source) +{ + if (!source) + return; + free(source->segments); + memset(source, 0, sizeof(*source)); +} + +static int64_t finish_mremap(mremap_source_t *source, int64_t result) +{ + dispose_mremap_source(source); + return result; +} + +/* Resolve a logical mremap source. Fork-aware growth intentionally leaves an + * inherited prefix and a child-private tail as separate records; those two + * records are still one VMA for mremap purposes. The stable vma_id proves that + * provenance even after another fork changes inherited_at_fork on both + * records. Reject any boundary with a different lineage so unrelated adjacent + * mappings cannot be copied as one source. + */ +static int find_mremap_source(const guest_t *g, + uint64_t start, + uint64_t size, + bool collect_segments, + mremap_source_t *source) +{ + uint64_t end = start + size; + const guest_region_t *first = guest_region_find(g, start); + if (!first) + return -LINUX_EFAULT; + + memset(source, 0, sizeof(*source)); + source->first = first; + + int index = (int) (first - g->regions); + if (index < 0 || index >= g->nregions) + goto invalid; + + uint64_t cursor = start; + uint64_t expected_gpa = first->gpa_base + (start - first->start); + uint64_t expected_offset = first->offset + (start - first->start); + uint64_t inherited_prefix = 0; + bool prefix_inherited = true; + int nsegments = 0; + for (int scan = index; scan < g->nregions && cursor < end; scan++) { + const guest_region_t *r = &g->regions[scan]; + if (r->start > cursor || r->end <= cursor) + goto invalid; + if (r != first) { + const guest_region_t *previous = &g->regions[scan - 1]; + if (previous->end != r->start || previous->prot != r->prot || + previous->flags != r->flags || previous->shared != r->shared || + previous->noreserve != r->noreserve || + previous->backing_ro != r->backing_ro || + strcmp(previous->name, r->name) != 0 || + !mremap_backings_match(previous, r)) + goto invalid; + if (!previous->vma_id || previous->vma_id != r->vma_id) + goto invalid; + } + + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + if (r->gpa_base + (cursor - r->start) != expected_gpa) + goto invalid; + if (!(r->flags & LINUX_MAP_ANONYMOUS) && + r->offset + (cursor - r->start) != expected_offset) + goto invalid; + + if (nsegments >= GUEST_MAX_REGIONS) + goto invalid; + nsegments++; + + if (prefix_inherited && r->inherited_at_fork) + inherited_prefix += segment_len; + else + prefix_inherited = false; + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + if (cursor != end) + goto invalid; + + /* Validation also gives the exact allocation size. Same-size non-fixed + * mremap stops here, so its no-op success cannot be turned into ENOMEM by + * segment bookkeeping. + */ + source->inherited_prefix = inherited_prefix; + source->nsegments = nsegments; + if (!collect_segments) + return 0; + + source->segments = malloc((size_t) nsegments * sizeof(*source->segments)); + if (!source->segments) { + dispose_mremap_source(source); + return -LINUX_ENOMEM; + } + + cursor = start; + expected_gpa = first->gpa_base + (start - first->start); + expected_offset = first->offset + (start - first->start); + for (int segment_index = 0; segment_index < nsegments; segment_index++) { + const guest_region_t *r = &g->regions[index + segment_index]; + uint64_t segment_end = r->end < end ? r->end : end; + uint64_t segment_len = segment_end - cursor; + mremap_source_segment_t *segment = &source->segments[segment_index]; + segment->start = cursor; + segment->end = segment_end; + segment->gpa_base = expected_gpa; + segment->offset = expected_offset; + segment->backing_fd = r->backing_fd; + segment->overlay_active = region_has_live_overlay(r); + segment->overlay_start = r->overlay_start; + segment->overlay_end = r->overlay_end; + + cursor = segment_end; + expected_gpa += segment_len; + expected_offset += segment_len; + } + return 0; + +invalid: + dispose_mremap_source(source); + return -LINUX_EFAULT; +} + +static int add_mremap_region(guest_t *g, + uint64_t start, + uint64_t old_size, + uint64_t new_size, + int prot, + int flags, + uint64_t offset, + const char *name, + int backing_fd, + bool inherited_at_fork, + uint64_t inherited_size, + int tail_backing_fd, + uint64_t vma_id) +{ + if (!inherited_at_fork) + inherited_size = 0; + if (inherited_size > old_size) + inherited_size = old_size; + if (inherited_size > new_size) + inherited_size = new_size; + + if (inherited_size > 0 && inherited_size < new_size) { + if (backing_fd >= 0 && tail_backing_fd < 0) { + close(backing_fd); + return -1; + } + if (guest_region_add_ex_owned(g, start, start + inherited_size, prot, + flags, offset, name, backing_fd, true, + vma_id) < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return -1; } + if (guest_region_add_ex_owned(g, start + inherited_size, + start + new_size, prot, flags, + offset + inherited_size, name, + tail_backing_fd, false, vma_id) < 0) { + guest_region_remove(g, start, start + inherited_size); + return -1; + } + return 0; } + + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return guest_region_add_ex_owned( + g, start, start + new_size, prot, flags, offset, name, backing_fd, + inherited_size == new_size && inherited_size > 0, vma_id); } static void region_clip_overlay(guest_region_t *r) @@ -241,43 +460,51 @@ static void region_clip_overlay(guest_region_t *r) region_clear_overlay(r); } -static void split_regions_at_boundary(guest_t *g, uint64_t boundary) +static int split_regions_at_boundary(guest_t *g, uint64_t boundary) { if (boundary == 0) - return; + return 0; for (int i = 0; i < g->nregions; i++) { - guest_region_t *r = &g->regions[i]; + const guest_region_t *r = &g->regions[i]; if (boundary <= r->start) break; if (boundary >= r->end) continue; if (g->nregions >= GUEST_MAX_REGIONS) { log_error( - "guest: region table full, cleanup split skipped at " + "guest: region table full, region split skipped at " "0x%llx", (unsigned long long) boundary); - return; + return -LINUX_ENOMEM; } + guest_region_t left = *r; + guest_region_t right = *r; + if (right.backing_fd >= 0) { + right.backing_fd = dup(right.backing_fd); + if (right.backing_fd < 0) { + log_error("guest: dup() failed for region split: %s", + strerror(errno)); + return -LINUX_ENOMEM; + } + } + + left.end = boundary; + right.offset += boundary - right.start; + right.gpa_base += boundary - right.start; + right.start = boundary; + region_clip_overlay(&left); + region_clip_overlay(&right); + memmove(&g->regions[i + 1], &g->regions[i], (g->nregions - i) * sizeof(guest_region_t)); + g->regions[i] = left; + g->regions[i + 1] = right; g->nregions++; - - g->regions[i].end = boundary; - g->regions[i + 1].offset += (boundary - g->regions[i + 1].start); - g->regions[i + 1].gpa_base += (boundary - g->regions[i + 1].start); - g->regions[i + 1].start = boundary; - if (g->regions[i + 1].backing_fd >= 0) { - g->regions[i + 1].backing_fd = dup(g->regions[i + 1].backing_fd); - if (g->regions[i + 1].backing_fd < 0) - log_error("guest: dup() failed for cleanup split: %s", - strerror(errno)); - } - region_clip_overlay(&g->regions[i]); - region_clip_overlay(&g->regions[i + 1]); - return; + return 0; } + return 0; } static uint64_t find_free_gap_inner(const guest_t *g, @@ -851,8 +1078,8 @@ static int64_t sys_mmap_high_va(guest_t *g, replaced_region_removed = true; } if (guest_region_add_ex_owned_gpa(g, addr, addr + length, gpa_base, prot, - flags, offset, NULL, - track_backing_fd) < 0) + flags, offset, NULL, track_backing_fd, + false, 0) < 0) goto fail; /* Ownership of track_backing_fd is now held by the new region. The fail * handler below skips closing when track_backing_fd < 0, so subsequent @@ -1052,6 +1279,167 @@ static int restore_file_overlay_range(guest_t *g, return 0; } +static bool mremap_source_has_overlay(const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + if (source->segments[i].overlay_active) + return true; + } + return false; +} + +typedef struct { + uint64_t start; + uint64_t end; + uint64_t offset; + int backing_fd; + bool active; + uint64_t overlay_start; + uint64_t overlay_end; +} saved_overlay_t; + +typedef void (*saved_overlay_getter_t)(const void *saved, + int index, + saved_overlay_t *overlay); + +static uint64_t saved_overlay_file_offset(const saved_overlay_t *overlay) +{ + if (overlay->overlay_start >= overlay->start) + return overlay->offset + (overlay->overlay_start - overlay->start); + return overlay->offset - (overlay->start - overlay->overlay_start); +} + +/* Reinstall each distinct host overlay once, then restore metadata on every + * tracker fragment that shared it. cleanup_overlays_in_range() can fail after + * removing only a subset, so replaying the complete saved set is rollback-safe. + */ +static int restore_saved_overlays_in_place(guest_t *g, + const void *saved, + int n, + saved_overlay_getter_t get) +{ + for (int i = 0; i < n; i++) { + saved_overlay_t overlay; + get(saved, i, &overlay); + if (!overlay.active || overlay.backing_fd < 0) + continue; + + uint64_t file_off = saved_overlay_file_offset(&overlay); + bool first = true; + for (int j = 0; j < i; j++) { + saved_overlay_t previous; + get(saved, j, &previous); + if (previous.active && previous.backing_fd >= 0 && + previous.overlay_start == overlay.overlay_start && + previous.overlay_end == overlay.overlay_end && + saved_overlay_file_offset(&previous) == file_off) { + first = false; + break; + } + } + + if (first) { + int err = restore_file_overlay_range( + g, overlay.start, overlay.end, overlay.overlay_start, + overlay.overlay_end, overlay.backing_fd, file_off); + if (err < 0) + return err; + } else { + mark_overlay_metadata_range(g, overlay.start, overlay.end, + overlay.overlay_start, + overlay.overlay_end); + } + } + return 0; +} + +static void get_mremap_source_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const mremap_source_t *source = saved; + const mremap_source_segment_t *segment = &source->segments[index]; + *overlay = (saved_overlay_t) { + .start = segment->start, + .end = segment->end, + .offset = segment->offset, + .backing_fd = segment->backing_fd, + .active = segment->overlay_active, + .overlay_start = segment->overlay_start, + .overlay_end = segment->overlay_end, + }; +} + +static int restore_mremap_source_overlays_in_place( + guest_t *g, + const mremap_source_t *source) +{ + return restore_saved_overlays_in_place(g, source, source->nsegments, + get_mremap_source_overlay); +} + +/* The host overlay stays installed during an in-place growth; only the region + * records are replaced. Reapply the saved metadata to the corresponding new + * records without remapping the host VA. + */ +static void mark_mremap_source_overlay_metadata(guest_t *g, + const mremap_source_t *source) +{ + for (int i = 0; i < source->nsegments; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + if (segment->overlay_active) + mark_overlay_metadata_range(g, segment->start, segment->end, + segment->overlay_start, + segment->overlay_end); + } +} + +/* Copy each source segment according to its own backing state. Live-overlay + * bytes must be refreshed from the file after the overlay is removed; private + * fork-grown bytes remain in the slab and must be copied from their GPA. + */ +static int copy_mremap_source(guest_t *g, + uint64_t dest_gpa, + uint64_t source_start, + uint64_t length, + const mremap_source_t *source) +{ + uint64_t source_end = source_start + length; + uint64_t cursor = source_start; + + for (int i = 0; i < source->nsegments && cursor < source_end; i++) { + const mremap_source_segment_t *segment = &source->segments[i]; + uint64_t start = segment->start > cursor ? segment->start : cursor; + uint64_t end = segment->end < source_end ? segment->end : source_end; + if (end <= start) + continue; + if (start != cursor) + return -LINUX_EFAULT; + + uint64_t len = end - start; + uint64_t dest = dest_gpa + (start - source_start); + if (segment->overlay_active) { + if (segment->backing_fd < 0) + return -LINUX_EFAULT; + int err = read_file_range_to_guest( + g, dest, segment->backing_fd, + segment->offset + (start - segment->start), len); + if (err < 0) + return err; + } else { + uint8_t *dest_ptr = host_ptr_for_gpa(g, dest); + uint8_t *source_ptr = host_ptr_for_gpa( + g, segment->gpa_base + (start - segment->start)); + if (!dest_ptr || !source_ptr) + return -LINUX_EFAULT; + memmove(dest_ptr, source_ptr, len); + } + cursor = end; + } + + return cursor == source_end ? 0 : -LINUX_EFAULT; +} + typedef struct { uint64_t overlay_start; uint64_t overlay_len; @@ -1099,8 +1487,12 @@ static int capture_region_snapshots(guest_t *g, region_snapshot_t *snaps, int max_snaps) { - split_regions_at_boundary(g, start); - split_regions_at_boundary(g, end); + int split_err = split_regions_at_boundary(g, start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, end); + if (split_err < 0) + return split_err; int n = 0; for (int i = 0; i < g->nregions; i++) { @@ -1118,6 +1510,7 @@ static int capture_region_snapshots(guest_t *g, snap->start = r->start; snap->end = r->end; snap->gpa_base = r->gpa_base; + snap->vma_id = r->vma_id; snap->prot = r->prot; snap->flags = r->flags; snap->offset = r->offset; @@ -1133,54 +1526,65 @@ static int capture_region_snapshots(guest_t *g, snap->overlay_start = r->overlay_start; snap->overlay_end = r->overlay_end; snap->backing_ro = r->backing_ro; + snap->inherited_at_fork = r->inherited_at_fork; str_copy_trunc(snap->name, r->name, sizeof(snap->name)); } return n; } -static int restore_snapshot_overlays_in_place(guest_t *g, - const region_snapshot_t *snaps, - int n) +/* MREMAP_FIXED may remove a destination fragment that used to share the same + * tracker backing fd as a source fragment. Rebind file-backed source segments + * to the owned source snapshots before destination removal, so later overlay + * restore and pread-based copies cannot observe a closed borrowed fd. */ +static int rebind_mremap_source_backings(mremap_source_t *source, + const region_snapshot_t *snaps, + int n) { - for (int i = 0; i < n; i++) { - const region_snapshot_t *snap = &snaps[i]; - if (!snap->overlay_active || snap->backing_fd < 0) + for (int i = 0; i < source->nsegments; i++) { + mremap_source_segment_t *segment = &source->segments[i]; + if (segment->backing_fd < 0) continue; - bool first = true; - uint64_t snap_file_off = - snap->offset + (snap->overlay_start - snap->start); - for (int j = 0; j < i; j++) { - const region_snapshot_t *prev = &snaps[j]; - if (!prev->overlay_active || prev->backing_fd < 0) - continue; - uint64_t prev_file_off = - prev->offset + (prev->overlay_start - prev->start); - if (prev->overlay_start == snap->overlay_start && - prev->overlay_end == snap->overlay_end && - prev_file_off == snap_file_off) { - first = false; + int stable_fd = -1; + for (int j = 0; j < n; j++) { + if (snaps[j].start <= segment->start && + segment->start < snaps[j].end && snaps[j].backing_fd >= 0) { + stable_fd = snaps[j].backing_fd; break; } } - - if (first) { - int err = restore_file_overlay_range( - g, snap->start, snap->end, snap->overlay_start, - snap->overlay_end, snap->backing_fd, snap_file_off); - if (err < 0) - return err; - continue; - } - - mark_overlay_metadata_range(g, snap->start, snap->end, - snap->overlay_start, snap->overlay_end); + if (stable_fd < 0) + return -LINUX_EFAULT; + segment->backing_fd = stable_fd; } - return 0; } +static void get_region_snapshot_overlay(const void *saved, + int index, + saved_overlay_t *overlay) +{ + const region_snapshot_t *snap = &((const region_snapshot_t *) saved)[index]; + *overlay = (saved_overlay_t) { + .start = snap->start, + .end = snap->end, + .offset = snap->offset, + .backing_fd = snap->backing_fd, + .active = snap->overlay_active, + .overlay_start = snap->overlay_start, + .overlay_end = snap->overlay_end, + }; +} + +static int restore_snapshot_overlays_in_place(guest_t *g, + const region_snapshot_t *snaps, + int n) +{ + return restore_saved_overlays_in_place(g, snaps, n, + get_region_snapshot_overlay); +} + static bool snapshot_has_materialized_ptes(const region_snapshot_t *snap) { return snap->prot != LINUX_PROT_NONE && @@ -1239,7 +1643,7 @@ static int restore_region_snapshots(guest_t *g, region_snapshot_t *snaps, int n) if (guest_region_add_ex_owned_gpa( g, snap->start, snap->end, snap->gpa_base, snap->prot, snap->flags, snap->offset, snap->name[0] ? snap->name : NULL, - snap->backing_fd) < 0) { + snap->backing_fd, snap->inherited_at_fork, snap->vma_id) < 0) { snap->backing_fd = -1; close_region_snapshots(snaps, n); return -LINUX_ENOMEM; @@ -1718,8 +2122,12 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) uint64_t host_start = ALIGN_DOWN(start, hps); uint64_t host_end = ALIGN_UP(end, hps); - split_regions_at_boundary(g, host_start); - split_regions_at_boundary(g, host_end); + int split_err = split_regions_at_boundary(g, host_start); + if (split_err < 0) + return split_err; + split_err = split_regions_at_boundary(g, host_end); + if (split_err < 0) + return split_err; /* Snapshot affected ranges first; the host-side mmap calls below do not * touch the region array, but a future caller invariant is to allow this @@ -1776,6 +2184,22 @@ static int cleanup_overlays_in_range(guest_t *g, uint64_t start, uint64_t end) /* Memory syscalls (tightly coupled to guest.h). */ +static bool heap_tail_can_extend(const guest_region_t *tail, + const guest_region_t *heap, + uint64_t old_brk) +{ + const int heap_flags = LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS; + + return tail->start == heap->end && tail->end == old_brk && + tail->end > tail->start && tail->gpa_base == tail->start && + tail->vma_id == heap->vma_id && + tail->prot == (LINUX_PROT_READ | LINUX_PROT_WRITE) && + tail->flags == heap_flags && tail->offset == 0 && + tail->backing_fd < 0 && !tail->shared && !tail->noreserve && + !tail->backing_ro && !tail->inherited_at_fork && + !region_has_live_overlay(tail) && !strcmp(tail->name, "[heap]"); +} + int64_t sys_brk(guest_t *g, uint64_t addr) { /* brk addresses as seen by the guest are IPA-based */ @@ -1797,6 +2221,16 @@ int64_t sys_brk(guest_t *g, uint64_t addr) return (int64_t) ipa_brk; } + /* Shrinking can split a file-backed semantic region when a fork child has + * grown and reshaped its heap metadata. Reserve the right-hand region's + * backing fd before publishing the new break, so EMFILE leaves both the + * break and region table unchanged. + */ + int shrink_remove_fd = -1; + if (new_off < old_brk && + guest_region_remove_prepare(g, new_off, old_brk, &shrink_remove_fd) < 0) + return (int64_t) ipa_brk; + /* Materialize any newly exposed heap pages. This must handle both: * 1. growth into brand-new 2 MiB blocks, and * 2. growth within an already-split block where finalize_block_perms() @@ -1824,12 +2258,53 @@ int64_t sys_brk(guest_t *g, uint64_t addr) * avoids the remove+add gap where a concurrent /proc/self/maps reader could * see no heap region. */ - if (new_off > g->brk_base) { + if (new_off < old_brk) { + /* Trim every semantic heap segment covered by the released suffix. + * In a fork child this may shorten or remove the private tail while + * leaving the inherited prefix intact. Keeping the tracker end equal + * to brk_current prevents a later regrowth from overlapping stale + * tail metadata. + */ + guest_region_remove_reserved(g, new_off, old_brk, shrink_remove_fd); + } else if (new_off > g->brk_base) { bool found = false; for (int i = 0; i < g->nregions; i++) { if (g->regions[i].start == g->brk_base && !strcmp(g->regions[i].name, "[heap]")) { - g->regions[i].end = new_off; + guest_region_t *heap = &g->regions[i]; + uint64_t old_heap_end = heap->end; + if (new_off > old_heap_end && heap->inherited_at_fork) { + /* Keep the fork-snapshot portion separate from pages + * materialized by post-fork brk growth. On later growths, + * extend the existing child-private tail rather than + * adding an overlapping range from the old boundary. + */ + guest_region_t *right = + i + 1 < g->nregions ? &g->regions[i + 1] : NULL; + guest_region_t *tail = + right && heap_tail_can_extend(right, heap, old_brk) + ? right + : NULL; + if (tail) { + if (new_off > tail->end) + tail->end = new_off; + } else if (new_off > old_brk && + guest_region_add_ex_owned( + g, old_brk, new_off, + LINUX_PROT_READ | LINUX_PROT_WRITE, + LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS, 0, + "[heap]", -1, false, heap->vma_id) < 0) { + /* Widening the inherited prefix would either overlap + * an incompatible child-private tail or mislabel new + * pages as inherited. Keep the original boundary; brk + * memory already grew successfully, so only the + * semantic tracker becomes stale. + */ + g->regions_tracker_stale = true; + } + } else { + heap->end = new_off; + } found = true; break; } @@ -2559,7 +3034,7 @@ int64_t sys_mmap(guest_t *g, */ if (guest_region_add_ex_owned(g, result_off, result_off + length, prot, track_flags, is_anon ? 0 : (uint64_t) offset, - NULL, track_backing_fd) < 0) { + NULL, track_backing_fd, false, 0) < 0) { /* Region table was full: undo any host overlay we just installed so the * file is not left mmap'd at host_base+ipa with no tracking. Without * this, a later operation in that range would memset zeros directly @@ -2689,12 +3164,20 @@ int64_t sys_mremap(guest_t *g, if (guest_range_hits_infra(g, old_off, old_off + old_size)) return -LINUX_EINVAL; - /* Verify the whole source range is covered by one tracked VMA. mremap() - * must not copy holes or unrelated adjacent mappings. + /* Verify the whole source range is covered by one logical VMA. A + * fork-aware growth can split that VMA at the inherited/private boundary, + * but no unrelated adjacent mapping may be included. */ - const guest_region_t *src_reg = guest_region_find(g, old_off); - if (!src_reg || src_reg->end - old_off < old_size) - return -LINUX_EFAULT; + mremap_source_t source; + bool collect_source_segments = + old_size != new_size || (flags & LINUX_MREMAP_FIXED); + int source_err = find_mremap_source(g, old_off, old_size, + collect_source_segments, &source); + if (source_err < 0) + return source_err; + const guest_region_t *src_reg = source.first; + uint64_t source_inherited_size = source.inherited_prefix; + uint64_t source_vma_id = src_reg->vma_id; /* Capture the source region's GPA layout before any region mutation below * invalidates src_reg. src_gpa_base + (va_off - src_start) is the backing @@ -2707,41 +3190,48 @@ int64_t sys_mremap(guest_t *g, /* Same size: nothing to do */ if (old_size == new_size && !(flags & LINUX_MREMAP_FIXED)) - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); /* Shrinking mremap keeps the base address and releases only the tail. */ if (new_size < old_size && !(flags & LINUX_MREMAP_FIXED)) { uint64_t tail_off = old_off + new_size, tail_end = old_off + old_size; + int tail_remove_fd = -1; + if (guest_region_remove_prepare(g, tail_off, tail_end, + &tail_remove_fd) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); /* Restore slab backing under any tail overlay before zeroing so the * memset does not write zeros into a file. */ int cleanup_err = cleanup_overlays_in_range(g, tail_off, tail_end); - if (cleanup_err < 0) - return cleanup_err; + if (cleanup_err < 0) { + if (tail_remove_fd >= 0) + close(tail_remove_fd); + return finish_mremap(&source, cleanup_err); + } /* Zero the trimmed region on its real backing (high-VA tails live at * gpa_base, not host_base + tail_off). */ memset(host_ptr_for_gpa(g, src_gpa_base + (tail_off - src_start)), 0, tail_end - tail_off); - guest_region_remove(g, tail_off, tail_end); + guest_region_remove_reserved(g, tail_off, tail_end, tail_remove_fd); guest_invalidate_ptes(g, tail_off, tail_end); if (tail_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = tail_off; if (tail_off < g->mmap_rx_gap_hint) g->mmap_rx_gap_hint = tail_off; - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } /* MREMAP_FIXED: move to a specific new address */ if (flags & LINUX_MREMAP_FIXED) { if (new_addr & 4095) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); uint64_t new_off = new_addr - g->ipa_base; /* MREMAP_FIXED dest stays primary-only for the same reason as the * source check above. */ if (new_off > g->guest_size || new_size > g->guest_size - new_off) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Same infrastructure protection as the source range: the move tail * removes any existing dest region and rewrites PTEs, which would @@ -2749,38 +3239,55 @@ int64_t sys_mremap(guest_t *g, * infra. */ if (guest_range_hits_infra(g, new_off, new_off + new_size)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Linux rejects MREMAP_FIXED when old and new ranges overlap */ uint64_t old_end = old_off + old_size, new_end = new_off + new_size; if (old_off < new_end && new_off < old_end) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); remove_range_t removed[] = { {old_off, old_end}, {new_off, new_end}, }; - if (!region_has_capacity_after_removes(g, removed, 2, 1)) - return -LINUX_ENOMEM; /* Capture old region metadata BEFORE modifying any regions. If mremap * removed destination first, an overlapping source would lose its * metadata. The overlap check above prevents this case, but capturing * first is still the safe ordering. */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; + int added_regions = + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; + if (!region_has_capacity_after_removes(g, removed, 2, added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } char track_name[sizeof(old_reg->name)] = {0}; /* Heap-allocated to avoid blowing the ~512 KiB default macOS thread * stack: each region_snapshot_t array is GUEST_MAX_REGIONS * @@ -2801,7 +3308,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } source_nsnaps = capture_region_snapshots( @@ -2811,7 +3320,20 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return source_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, source_nsnaps); + } + int rebind_err = + rebind_mremap_source_backings(&source, source_snaps, source_nsnaps); + if (rebind_err < 0) { + dispose_region_snapshots(&source_snaps, &source_nsnaps); + free(dest_snaps); + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, rebind_err); } dest_nsnaps = capture_region_snapshots(g, new_off, new_off + new_size, dest_snaps, GUEST_MAX_REGIONS); @@ -2820,7 +3342,9 @@ int64_t sys_mremap(guest_t *g, free(dest_snaps); if (track_backing_fd >= 0) close(track_backing_fd); - return dest_nsnaps; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, dest_nsnaps); } if (source_overlay) { @@ -2833,7 +3357,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } } @@ -2847,7 +3373,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2855,7 +3383,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, cleanup_err); } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { @@ -2866,7 +3396,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, restore_err); } (void) restore_snapshot_overlays_in_place(g, dest_snaps, dest_nsnaps); @@ -2874,7 +3406,9 @@ int64_t sys_mremap(guest_t *g, dispose_region_snapshots(&source_snaps, &source_nsnaps); if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } /* Remove existing mappings at the destination after all fallible @@ -2882,19 +3416,19 @@ int64_t sys_mremap(guest_t *g, */ guest_region_remove(g, new_off, new_off + new_size); - /* Copy data (use memmove for potential overlap). If the source has a - * live overlay, the read side of the memmove pulls live file content; - * the destination receives a private snapshot at mremap time (no + /* Copy each logical source segment according to its own backing state. + * The destination receives a private snapshot at mremap time (no * overlay reapplied), and msync's emulated pwrite-the-diff path keeps * subsequent writes consistent. */ uint64_t copy_len = old_size < new_size ? old_size : new_size; if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, copy_len); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, copy_len, &source); if (copy_err < 0) { int restore_err = restore_snapshot_overlays_in_place( g, source_snaps, source_nsnaps); @@ -2915,21 +3449,14 @@ int64_t sys_mremap(guest_t *g, restore_err = pt_err; if (track_backing_fd >= 0) close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); if (restore_err < 0) - return restore_err; - return copy_err; + return finish_mremap(&source, restore_err); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA (identity for primary sources, - * overflow/mapping backing for high-VA). The destination is always - * a fresh primary-window range, so it never overlaps the source and - * the copy direction does not matter. - */ - memmove((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - copy_len); } /* Zero any extension beyond old data */ if (new_size > old_size) @@ -2948,19 +3475,21 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; } - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) { + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) { (void) restore_region_snapshots(g, dest_snaps, dest_nsnaps); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); dispose_region_snapshots(&source_snaps, &source_nsnaps); dispose_region_snapshots(&dest_snaps, &dest_nsnaps); - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Grow in place: try to extend without moving */ @@ -2973,7 +3502,7 @@ int64_t sys_mremap(guest_t *g, * it. */ if (guest_range_hits_infra(g, grow_off, grow_off + grow_len)) - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); /* Check if the space after the old region is free (overflow-safe) */ if (grow_off <= g->guest_size && grow_len <= g->guest_size - grow_off) { @@ -2993,26 +3522,55 @@ int64_t sys_mremap(guest_t *g, if (can_grow) { remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) - return -LINUX_ENOMEM; - /* Extend in place */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); - bool old_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t old_overlay_start = - old_overlay ? old_reg->overlay_start : 0; - uint64_t old_overlay_end = - old_overlay ? old_reg->overlay_end : 0; + int tail_backing_fd = -1; + if (source_inherited_size > 0 && + source_inherited_size < new_size && track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } bool old_backing_ro = old_reg && old_reg->backing_ro; - if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; + bool old_inherited_at_fork = + old_reg && old_reg->inherited_at_fork; + int added_regions = source_inherited_size > 0 && + source_inherited_size < new_size + ? 2 + : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, + added_regions)) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + if (old_reg && old_reg->backing_fd >= 0 && + track_backing_fd < 0) { + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, @@ -3021,22 +3579,26 @@ int64_t sys_mremap(guest_t *g, if (mremap_extend_range(g, grow_off, grow_len, prot) < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } memset((uint8_t *) g->host_base + grow_off, 0, grow_len); /* Update region tracking: remove old, add extended */ - guest_region_remove(g, old_off, old_off + old_size); - if (guest_region_add_ex_owned(g, old_off, old_off + new_size, - prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, - track_backing_fd) < 0) - return -LINUX_ENOMEM; - if (old_overlay) - mark_overlay_metadata_range(g, old_off, old_off + old_size, - old_overlay_start, - old_overlay_end); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); + if (add_mremap_region(g, old_off, old_size, new_size, prot, + track_flags, track_offset, + track_name[0] ? track_name : NULL, + track_backing_fd, old_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); + mark_mremap_source_overlay_metadata(g, &source); if (old_backing_ro) mark_region_backing_ro(g, old_off, old_off + new_size); @@ -3050,35 +3612,37 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) old_addr; + return finish_mremap(&source, (int64_t) old_addr); } } /* Growth in place failed; MREMAP_MAYMOVE is required */ if (!(flags & LINUX_MREMAP_MAYMOVE)) - return -LINUX_ENOMEM; + return finish_mremap(&source, -LINUX_ENOMEM); /* Allocate a new region and move */ - const guest_region_t *old_reg = guest_region_find(g, old_off); + const guest_region_t *old_reg = src_reg; int prot = old_reg ? old_reg->prot : (LINUX_PROT_READ | LINUX_PROT_WRITE); int track_flags = old_reg ? old_reg->flags : (LINUX_MAP_PRIVATE | LINUX_MAP_ANONYMOUS); - uint64_t track_offset = old_reg ? old_reg->offset : 0; + uint64_t track_offset = + old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; int track_backing_fd = dup_region_backing_fd(old_reg); if (old_reg && old_reg->backing_fd >= 0 && track_backing_fd < 0) - return -LINUX_ENOMEM; - bool source_overlay = old_reg && region_has_live_overlay(old_reg); - uint64_t source_overlay_start = - source_overlay ? old_reg->overlay_start : 0; - uint64_t source_overlay_end = source_overlay ? old_reg->overlay_end : 0; + return finish_mremap(&source, -LINUX_ENOMEM); + int tail_backing_fd = -1; + if (source_inherited_size > 0 && source_inherited_size < new_size && + track_backing_fd >= 0) { + tail_backing_fd = dup(track_backing_fd); + if (tail_backing_fd < 0) { + close(track_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + } + bool source_overlay = mremap_source_has_overlay(&source); bool source_backing_ro = old_reg && old_reg->backing_ro; - uint64_t source_file_off = - old_reg ? old_reg->offset + (old_off - old_reg->start) : 0; - uint64_t source_overlay_file_off = - source_overlay - ? old_reg->offset + (source_overlay_start - old_reg->start) - : 0; + bool source_inherited_at_fork = old_reg && old_reg->inherited_at_fork; char track_name[sizeof(old_reg->name)] = {0}; if (old_reg) str_copy_trunc(track_name, old_reg->name, sizeof(track_name)); @@ -3100,54 +3664,85 @@ int64_t sys_mremap(guest_t *g, if (new_off == UINT64_MAX) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } remove_range_t removed = {old_off, old_off + old_size}; - if (!region_has_capacity_after_removes(g, &removed, 1, 1)) { + int added_regions = + source_inherited_size > 0 && source_inherited_size < new_size ? 2 + : 1; + if (!region_has_capacity_after_removes(g, &removed, 1, added_regions)) { if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); + } + + int source_remove_fd = -1; + if (guest_region_remove_prepare(g, old_off, old_off + old_size, + &source_remove_fd) < 0) { + if (track_backing_fd >= 0) + close(track_backing_fd); + if (tail_backing_fd >= 0) + close(tail_backing_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } if (source_overlay) { int cleanup_err = cleanup_overlays_in_range(g, old_off, old_off + old_size); if (cleanup_err < 0) { + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (track_backing_fd >= 0) close(track_backing_fd); - return cleanup_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + if (restore_err < 0) + return finish_mremap(&source, restore_err); + return finish_mremap(&source, cleanup_err); } } if (mremap_extend_range(g, new_off, new_size, prot) < 0) { if (source_overlay) { - int restore_err = restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + int restore_err = + restore_mremap_source_overlays_in_place(g, &source); if (restore_err < 0) { if (track_backing_fd >= 0) close(track_backing_fd); - return restore_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, restore_err); } } if (track_backing_fd >= 0) close(track_backing_fd); - return -LINUX_ENOMEM; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, -LINUX_ENOMEM); } - /* Copy old data, zero extension. The new range was just allocated from - * a free gap so it has no overlays to clean up; the source may have an - * overlay, which is read transparently by the memcpy before its - * underlying slab is restored below. + /* Copy each source segment according to its own backing state, then + * zero the extension. The new range is a fresh gap and receives no + * live overlay. */ if (prot == LINUX_PROT_NONE) { memset((uint8_t *) g->host_base + new_off, 0, new_size); - } else if (source_overlay) { - memset((uint8_t *) g->host_base + new_off, 0, new_size); - int copy_err = read_file_range_to_guest( - g, new_off, track_backing_fd, source_file_off, old_size); + } else { + if (source_overlay) + memset((uint8_t *) g->host_base + new_off, 0, new_size); + int copy_err = + copy_mremap_source(g, new_off, old_off, old_size, &source); if (copy_err < 0) { /* Roll back both sides: re-apply the source overlay so the * caller's MAP_SHARED is not silently demoted to a slab @@ -3155,23 +3750,16 @@ int64_t sys_mremap(guest_t *g, * allocated via mremap_extend_range so the guest does not see * phantom zero pages where the failed mremap landed. */ - (void) restore_file_overlay_range( - g, old_off, old_off + old_size, source_overlay_start, - source_overlay_end, track_backing_fd, - source_overlay_file_off); + (void) restore_mremap_source_overlays_in_place(g, &source); guest_invalidate_ptes(g, new_off, new_off + new_size); if (track_backing_fd >= 0) close(track_backing_fd); - return copy_err; + if (tail_backing_fd >= 0) + close(tail_backing_fd); + if (source_remove_fd >= 0) + close(source_remove_fd); + return finish_mremap(&source, copy_err); } - } else { - /* Read the source through its GPA so high-VA sources copy from - * their real backing (identity for primary: == host_base + - * old_off). The destination is a fresh primary-window gap. - */ - memcpy((uint8_t *) g->host_base + new_off, - host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), - old_size); } memset((uint8_t *) g->host_base + new_off + old_size, 0, new_size - old_size); @@ -3181,7 +3769,8 @@ int64_t sys_mremap(guest_t *g, */ memset(host_ptr_for_gpa(g, src_gpa_base + (old_off - src_start)), 0, old_size); - guest_region_remove(g, old_off, old_off + old_size); + guest_region_remove_reserved(g, old_off, old_off + old_size, + source_remove_fd); guest_invalidate_ptes(g, old_off, old_off + old_size); if (old_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = old_off; @@ -3189,10 +3778,12 @@ int64_t sys_mremap(guest_t *g, g->mmap_rx_gap_hint = old_off; /* Track new region */ - if (guest_region_add_ex_owned( - g, new_off, new_off + new_size, prot, track_flags, track_offset, - track_name[0] ? track_name : NULL, track_backing_fd) < 0) - return -LINUX_ENOMEM; + if (add_mremap_region(g, new_off, old_size, new_size, prot, track_flags, + track_offset, track_name[0] ? track_name : NULL, + track_backing_fd, source_inherited_at_fork, + source_inherited_size, tail_backing_fd, + source_vma_id) < 0) + return finish_mremap(&source, -LINUX_ENOMEM); if (source_backing_ro) mark_region_backing_ro(g, new_off, new_off + new_size); @@ -3206,11 +3797,11 @@ int64_t sys_mremap(guest_t *g, g->mmap_next = hwm; } - return (int64_t) guest_ipa(g, new_off); + return finish_mremap(&source, (int64_t) guest_ipa(g, new_off)); } /* Should not reach here */ - return -LINUX_EINVAL; + return finish_mremap(&source, -LINUX_EINVAL); } /* sys_madvise. */ @@ -3412,20 +4003,34 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) if (guest_range_hits_infra(g, unmap_off, end)) return -LINUX_EINVAL; + /* An interior removal from a file-backed region needs a second owned fd + * for the surviving right half. Reserve it before changing overlays, + * page tables, or host memory so descriptor exhaustion is failure-atomic. + */ + int remove_fd = -1; + if (guest_region_remove_prepare(g, unmap_off, end, &remove_fd) < 0) + return -LINUX_ENOMEM; + /* Restore slab backing under any active MAP_SHARED file overlay before * zeroing the host VA. Without this, the memset below would write zeros * directly into the file. */ int cleanup_err = cleanup_overlays_in_range(g, unmap_off, end); - if (cleanup_err < 0) + if (cleanup_err < 0) { + if (remove_fd >= 0) + close(remove_fd); return cleanup_err; + } /* Invalidate PTEs first. This may need to split a 2MiB block which can fail * if the page table pool is exhausted. Failing before region removal keeps * metadata consistent. */ - if (guest_invalidate_ptes(g, unmap_off, end) < 0) + if (guest_invalidate_ptes(g, unmap_off, end) < 0) { + if (remove_fd >= 0) + close(remove_fd); return -LINUX_ENOMEM; + } for (int i = 0; i < g->nregions; i++) { guest_region_t *r = &g->regions[i]; if (r->start >= end) @@ -3438,7 +4043,7 @@ static int munmap_guest_range(guest_t *g, uint64_t unmap_off, uint64_t end) uint64_t zend = (r->end < end) ? r->end : end; memset((uint8_t *) g->host_base + zstart, 0, zend - zstart); } - guest_region_remove(g, unmap_off, end); + guest_region_remove_reserved(g, unmap_off, end, remove_fd); if (unmap_off < g->mmap_rw_gap_hint) g->mmap_rw_gap_hint = unmap_off; if (unmap_off < g->mmap_rx_gap_hint) @@ -3492,9 +4097,16 @@ int64_t sys_munmap(guest_t *g, uint64_t addr, uint64_t length) if (addr <= 0x0000FFFFFFFFFFFFULL) { if (addr >= g->guest_size) { if (region_range_overlaps(g, addr, addr + length)) { - if (guest_invalidate_ptes(g, addr, addr + length) < 0) + int remove_fd = -1; + if (guest_region_remove_prepare(g, addr, addr + length, + &remove_fd) < 0) return -LINUX_ENOMEM; - guest_region_remove(g, addr, addr + length); + if (guest_invalidate_ptes(g, addr, addr + length) < 0) { + if (remove_fd >= 0) + close(remove_fd); + return -LINUX_ENOMEM; + } + guest_region_remove_reserved(g, addr, addr + length, remove_fd); } return 0; } diff --git a/tests/manifest.txt b/tests/manifest.txt index d33d9527..552a01e0 100644 --- a/tests/manifest.txt +++ b/tests/manifest.txt @@ -109,6 +109,7 @@ test-mmap-hint [section] mremap tests test-mremap test-mremap-infra +test-mremap-fork-tracking test-shim-cred-race [section] msync MAP_SHARED tests diff --git a/tests/test-dev-shm-paths.c b/tests/test-dev-shm-paths.c index 728aae9d..dff9e9ff 100644 --- a/tests/test-dev-shm-paths.c +++ b/tests/test-dev-shm-paths.c @@ -51,19 +51,41 @@ static char shm_dir[128]; static char shm_fifo[128]; static char shm_exec[128]; static char victim_path[128]; +static char fixture_suffix[16]; -static void name_fixtures(void) +/* Guest PIDs restart from the same value in each independent elfuse process. + * Reserve a host-unique suffix so concurrent runtime jobs cannot unlink or + * replace one another's /dev/shm fixtures. */ +static int name_fixtures(void) { - int pid = (int) getpid(); - snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%d", pid); - snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%d", pid); - snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%d", pid); - snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%d", pid); - snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%d", pid); - snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%d", pid); - snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%d", pid); - snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%d", - pid); + char seed[] = "/tmp/elfuse-shm-seed-XXXXXX"; + int fd = mkstemp(seed); + if (fd < 0) + return -1; + close(fd); + (void) unlink(seed); + + const char *suffix = strrchr(seed, '-'); + if (suffix == NULL || suffix[1] == '\0') + return -1; + snprintf(fixture_suffix, sizeof(fixture_suffix), "%s", suffix + 1); + + snprintf(shm_path, sizeof(shm_path), SHM_DIR "elfuse_paths_%s", + fixture_suffix); + snprintf(shm_path2, sizeof(shm_path2), SHM_DIR "elfuse_paths2_%s", + fixture_suffix); + snprintf(shm_link, sizeof(shm_link), SHM_DIR "elfuse_link_%s", + fixture_suffix); + snprintf(shm_evil, sizeof(shm_evil), SHM_DIR "elfuse_evil_%s", + fixture_suffix); + snprintf(shm_dir, sizeof(shm_dir), SHM_DIR "elfuse_dir_%s", fixture_suffix); + snprintf(shm_fifo, sizeof(shm_fifo), SHM_DIR "elfuse_fifo_%s", + fixture_suffix); + snprintf(shm_exec, sizeof(shm_exec), SHM_DIR "elfuse_exec_%s", + fixture_suffix); + snprintf(victim_path, sizeof(victim_path), "/tmp/elfuse-shm-victim-%s", + fixture_suffix); + return 0; } static void cleanup_fixtures(void) @@ -724,7 +746,7 @@ static void test_dotdot_bearing_names_allowed(void) static const char *names[] = {"a..b", "..a", "a..", "..."}; for (size_t i = 0; i < sizeof(names) / sizeof(names[0]); i++) { char p[128]; - snprintf(p, sizeof(p), SHM_DIR "elfuse_%d_%s", (int) getpid(), + snprintf(p, sizeof(p), SHM_DIR "elfuse_%s_%s", fixture_suffix, names[i]); int fd = open(p, O_CREAT | O_EXCL | O_RDWR, 0600); if (fd < 0) { @@ -775,7 +797,11 @@ int main(int argc, char **argv) printf("test-dev-shm-paths: /dev/shm path-syscall consistency\n"); - name_fixtures(); + if (name_fixtures() < 0) { + FAIL("reserve unique fixture suffix"); + SUMMARY("test-dev-shm-paths"); + return 1; + } cleanup_fixtures(); if (test_open_then_chmod() == 0) { diff --git a/tests/test-dynamic-array-host.c b/tests/test-dynamic-array-host.c new file mode 100644 index 00000000..0ca09eb1 --- /dev/null +++ b/tests/test-dynamic-array-host.c @@ -0,0 +1,145 @@ +/* + * Native-host unit tests for the generic dynamic array. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include + +#include "dynamic-array.h" + +typedef struct odd { + unsigned char tag; + uint32_t value; +} odd_t; + +DYNAMIC_ARRAY_DEFINE(int_array, int) +DYNAMIC_ARRAY_DEFINE(odd_array, odd_t) + +static void test_zero_and_init(void) +{ + int_array_t fresh; + assert(int_array_init_with_capacity(&fresh, 4) == 0); + assert(int_array_capacity(&fresh) >= 4); + int_array_destroy(&fresh); + + int_array_t fresh_lazy; + assert(int_array_init(&fresh_lazy) == 0); + assert(int_array_append_value(&fresh_lazy, 11) == 0); + assert(*int_array_at(&fresh_lazy, 0) == 11); + int_array_destroy(&fresh_lazy); + + int_array_t values = {0}; + int value = 7; + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + assert(int_array_append(&values, &value) == 0); + assert(int_array_count(&values) == 1); + assert(*int_array_at(&values, 0) == 7); + int_array_destroy(&values); + assert(int_array_data(&values) == NULL); + assert(int_array_count(&values) == 0); + assert(int_array_capacity(&values) == 0); + + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_capacity(&values) >= 4); + int_array_destroy(&values); +} + +static void test_growth_insert_and_stride(void) +{ + odd_array_t values = {0}; + odd_t first = {1, 11}; + odd_t second = {2, 22}; + odd_t middle = {3, 33}; + assert(odd_array_append(&values, &first) == 0); + assert(odd_array_append(&values, &second) == 0); + assert(odd_array_insert(&values, 1, &middle) == 0); + assert(odd_array_count(&values) == 3); + assert(odd_array_at(&values, 1)->tag == 3); + assert(odd_array_at(&values, 2)->value == 22); + assert(odd_array_capacity(&values) >= 3); + odd_array_destroy(&values); +} + +static void test_alias_and_resize(void) +{ + int_array_t values = {0}; + int initial[] = {1, 2, 3, 4}; + assert(int_array_init_with_capacity(&values, 4) == 0); + assert(int_array_append_n(&values, initial, 4) == 0); + int *old_data = int_array_data(&values); + int *alias = int_array_at(&values, 1); + assert(int_array_append(&values, alias) == 0); + assert(int_array_data(&values) != old_data || + int_array_capacity(&values) > 4); + assert(int_array_count(&values) == 5); + assert(*int_array_at(&values, 4) == 2); + assert(int_array_resize(&values, 8) == 0); + for (size_t i = 5; i < 8; i++) + assert(*int_array_at(&values, i) == 0); + assert(int_array_resize(&values, 2) == 0); + int_array_destroy(&values); +} + +static void test_invalid_and_overflow(void) +{ + int_array_t values = {0}; + int value = 9; + errno = 0; + assert(int_array_insert(&values, 1, &value) == -1); + assert(errno == EINVAL); + errno = 0; + assert(int_array_append_n(&values, &value, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + + dynamic_array_t raw = {0}; + errno = 0; + assert(dynamic_array_init(&raw, 0) == -1); + assert(errno == EINVAL); + assert(dynamic_array_init(&raw, sizeof(uint64_t)) == 0); + errno = 0; + assert(dynamic_array_reserve(&raw, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(raw.data == NULL && raw.count == 0 && raw.capacity == 0); + dynamic_array_destroy(&raw); + + /* A malformed metadata state must not let failed byte-size calculations + * feed uninitialized offsets into memory operations. */ + dynamic_array_t resize_overflow = { + .data = NULL, + .count = 0, + .capacity = 2, + .element_size = SIZE_MAX, + }; + errno = 0; + assert(dynamic_array_resize(&resize_overflow, 2) == -1); + assert(errno == EOVERFLOW); + assert(resize_overflow.count == 0); + + dynamic_array_t append_overflow = { + .data = NULL, + .count = SIZE_MAX / 2 + 1, + .capacity = SIZE_MAX, + .element_size = 2, + }; + errno = 0; + assert(dynamic_array_append(&append_overflow, &value) == -1); + assert(errno == EOVERFLOW); + assert(append_overflow.count == SIZE_MAX / 2 + 1); +} + +int main(void) +{ + test_zero_and_init(); + test_growth_insert_and_stride(); + test_alias_and_resize(); + test_invalid_and_overflow(); + puts("test-dynamic-array-host: PASS"); + return 0; +} diff --git a/tests/test-fork-ipc-protocol-host.c b/tests/test-fork-ipc-protocol-host.c index 8dc4cc88..0f420098 100644 --- a/tests/test-fork-ipc-protocol-host.c +++ b/tests/test-fork-ipc-protocol-host.c @@ -19,9 +19,10 @@ #define LEGACY_ELFK_MAGIC 0x454C464BU #define PREVIOUS_ELFL_MAGIC 0x454C464CU #define PREVIOUS_ELFM_MAGIC 0x454C464DU +#define PREVIOUS_ELFN_MAGIC 0x454C464EU -_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464EU, - "fork IPC protocol magic must remain ELFN until the next " +_Static_assert(FORK_IPC_PROTOCOL_MAGIC == 0x454C464FU, + "fork IPC protocol magic must remain ELFO until the next " "incompatible wire-format change"); _Static_assert(IPC_MAGIC_HEADER == FORK_IPC_PROTOCOL_MAGIC, "header magic must be the protocol identity"); @@ -31,6 +32,8 @@ _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFL_MAGIC, "NOFILE header fields require rejecting ELFL peers"); _Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFM_MAGIC, "start_stack header field requires rejecting ELFM peers"); +_Static_assert(FORK_IPC_PROTOCOL_MAGIC != PREVIOUS_ELFN_MAGIC, + "region fork metadata requires rejecting ELFN peers"); _Static_assert(IPC_MAGIC_SENTINEL != FORK_IPC_PROTOCOL_MAGIC, "process-state sentinel must not alias the header protocol"); diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 8bde500a..fcddc3ee 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -243,10 +243,10 @@ unstage_sysroot_fixtures() # Generic test helpers. -# The qemu reference lane now runs every matrix test against the real Alpine -# linux-virt kernel, so QEMU_SKIP is empty. Add a test's name here only if it -# asserts elfuse-specific behavior a real kernel does not honor; it still runs -# in elfuse-aarch64 mode and in 'make check'. +# The qemu reference lane runs the portable matrix tests against the real +# Alpine linux-virt kernel. Add a test's name here only if it asserts +# elfuse-specific behavior a real kernel does not honor; it still runs in +# elfuse-aarch64 mode and in 'make check'. # # The two oom_adj/oom_score_adj sendfile-and-copy_file_range-interception # subtests that used to make test-io-opt diverge here were split out into @@ -283,6 +283,7 @@ QEMU_SKIP=" test-fd-family test-scm-creds test-proc-fidelity + test-proc-smap " # test-session: getpgid/getsid/setsid assume the test is its own session and # process-group leader, true when elfuse launches it directly but not when @@ -352,6 +353,10 @@ QEMU_SKIP=" # under elfuse, while a real kernel allows the open and only rejects the # write -- a genuine behavioral difference worth reviewing on its own, # not just an environment artifact. +# test-proc-smap: validates elfuse's synthetic smaps VMA snapshot, including +# per-VMA Shared_Dirty inheritance and exclusion of post-fork VMAs. Real +# Linux smaps exposes kernel-owned VMA/page accounting instead, so this is +# intentionally not a reference-kernel invariant. is_qemu_skipped() { @@ -667,6 +672,7 @@ run_unit_tests() test_rc "$runner" "test-procfs-exec" 0 "$bindir/test-procfs-exec" test_rc "$runner" "test-proc-limits" 0 "$bindir/test-proc-limits" test_rc "$runner" "test-proc-fidelity" 0 "$bindir/test-proc-fidelity" + test_rc "$runner" "test-proc-smap" 0 "$bindir/test-proc-smap" printf "\nNetwork\n" test_check "$runner" "test-net" "0 failed" "$bindir/test-net" @@ -1231,7 +1237,7 @@ run_suite() # observed counts diverge. apple-unknown is the fallback row for SoC strings the # detector does not recognize yet. EXPECTED_BASELINES=( - "elfuse-aarch64|238|0" + "elfuse-aarch64|239|0" "qemu-aarch64|218|0" "elfuse-x86_64:apple-m1-m2|71|0" "elfuse-x86_64:apple-m3-plus|71|0" diff --git a/tests/test-mremap-fork-tracking.c b/tests/test-mremap-fork-tracking.c new file mode 100644 index 00000000..0bdf9b31 --- /dev/null +++ b/tests/test-mremap-fork-tracking.c @@ -0,0 +1,842 @@ +/* + * elfuse-internal mremap fork-tracking tests + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * These cases exercise elfuse's inherited-at-fork region bookkeeping, not a + * portable Linux ABI contract. Do not add this binary to test-matrix.sh: + * Linux can merge the adjacent file mappings below, and extends MAP_SHARED + * mappings directly from the backing file rather than creating elfuse's + * child-private tracking tail. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif +#ifndef MREMAP_FIXED +#define MREMAP_FIXED 2 +#endif + +static bool set_program_break(uintptr_t address) +{ + return brk((void *) address) == 0 && sbrk(0) == (void *) address; +} + +static ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) +{ + if (bufsz == 0) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + ssize_t total = 0; + while ((size_t) total < bufsz - 1) { + ssize_t n = read(fd, buf + total, bufsz - 1 - (size_t) total); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + int saved_errno = errno; + (void) close(fd); + errno = saved_errno; + return -1; + } + if (n == 0) + break; + total += n; + } + buf[total] = '\0'; + (void) close(fd); + return total; +} + +static void *reserve_then_map_fixed(size_t reserve_length, + size_t mapping_length, + int prot, + int flags, + int fd, + off_t offset) +{ + void *base = mmap(NULL, reserve_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) + return MAP_FAILED; + if (munmap(base, reserve_length) != 0) { + int saved_errno = errno; + (void) munmap(base, reserve_length); + errno = saved_errno; + return MAP_FAILED; + } + return mmap(base, mapping_length, prot, flags | MAP_FIXED, fd, offset); +} + +static bool parse_maps_hex(const char **cursor, + const char *end, + char delimiter, + uintptr_t *value) +{ + const char *start = *cursor; + uintptr_t parsed = 0; + while (*cursor < end && **cursor != delimiter) { + unsigned int digit; + char c = **cursor; + if (c >= '0' && c <= '9') + digit = (unsigned int) (c - '0'); + else if (c >= 'a' && c <= 'f') + digit = (unsigned int) (c - 'a' + 10); + else + return false; + parsed = parsed * 16 + digit; + (*cursor)++; + } + if (*cursor == start || *cursor >= end) + return false; + *value = parsed; + return true; +} + +static bool mapping_is_rw_private(uintptr_t address) +{ + char maps[64 * 1024]; + ssize_t length = read_file_nul("/proc/self/maps", maps, sizeof(maps)); + if (length <= 0) + return false; + + const char *cursor = maps; + const char *end = maps + length; + while (cursor < end) { + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&cursor, end, '-', &start)) + return false; + cursor++; + if (!parse_maps_hex(&cursor, end, ' ', &limit)) + return false; + cursor++; + if (cursor + 4 > end) + return false; + if (start <= address && address < limit) { + return cursor[0] == 'r' && cursor[1] == 'w' && cursor[2] == '-' && + cursor[3] == 'p'; + } + while (cursor < end && *cursor != '\n') + cursor++; + if (cursor < end) + cursor++; + } + return false; +} + +static char maps_buffer[1024 * 1024]; + +static bool read_maps_snapshot(ssize_t *length_out) +{ + ssize_t length = + read_file_nul("/proc/self/maps", maps_buffer, sizeof(maps_buffer)); + if (length <= 0 || (size_t) length == sizeof(maps_buffer) - 1) + return false; + *length_out = length; + return true; +} + +static bool count_maps_entries(int *count_out) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + int count = 0; + for (ssize_t i = 0; i < length; i++) { + if (maps_buffer[i] == '\n') + count++; + } + *count_out = count; + return true; +} + +static bool heap_headers_do_not_overlap(int *heap_count) +{ + ssize_t length = 0; + if (!read_maps_snapshot(&length)) + return false; + + const char heap_name[] = "[heap]"; + const char *cursor = maps_buffer; + const char *end = maps_buffer + length; + uintptr_t previous_end = 0; + int count = 0; + + while (cursor < end) { + const char *line = cursor; + while (cursor < end && *cursor != '\n') + cursor++; + const char *line_end = cursor; + if (cursor < end) + cursor++; + + bool is_heap = false; + for (const char *p = line; p + sizeof(heap_name) - 1 <= line_end; p++) { + if (memcmp(p, heap_name, sizeof(heap_name) - 1) == 0) { + is_heap = true; + break; + } + } + if (!is_heap) + continue; + + const char *field = line; + uintptr_t start = 0, limit = 0; + if (!parse_maps_hex(&field, line_end, '-', &start)) + return false; + field++; + if (!parse_maps_hex(&field, line_end, ' ', &limit)) + return false; + if (count > 0 && start < previous_end) + return false; + previous_end = limit; + count++; + } + + *heap_count = count; + return true; +} + +static void test_postfork_adjacent_anon_rejected(void) +{ + TEST("mremap rejects unrelated post-fork tail"); + + const size_t span = 64 * 1024; + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (first == MAP_FAILED) { + FAIL("inherited anonymous mmap failed"); + return; + } + void *base = first; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(first, span); + return; + } + if (pid == 0) { + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (second != (char *) base + span) + _exit(30); + + errno = 0; + void *shrunk = mremap(base, 2 * span, span, 0); + if (shrunk == MAP_FAILED && errno == EFAULT) + _exit(0); + _exit(31); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("mremap accepted unrelated post-fork mapping"); + + munmap(first, span); +} + +static void test_adjacent_file_vmas_rejected(void) +{ + TEST("mremap rejects adjacent independent file VMAs"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-adjacent-XXXXXX"; + int fd1 = mkstemp(tmpl); + if (fd1 < 0) { + FAIL("mkstemp failed"); + return; + } + int fd2 = open(tmpl, O_RDWR); + unlink(tmpl); + if (fd2 < 0 || ftruncate(fd1, (off_t) (2 * span)) != 0) { + FAIL("file setup failed"); + if (fd2 >= 0) + close(fd2); + close(fd1); + return; + } + + void *first = reserve_then_map_fixed(2 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd1, 0); + void *base = first; + if (first == MAP_FAILED) { + FAIL("address reservation or first file mmap failed"); + close(fd2); + close(fd1); + return; + } + void *second = mmap((char *) base + span, span, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, fd2, (off_t) span); + if (first != base || second != (char *) base + span) { + FAIL("adjacent file mmap failed"); + if (first == base) + munmap(first, span); + if (second == (char *) base + span) + munmap(second, span); + close(fd2); + close(fd1); + return; + } + + errno = 0; + void *q = mremap(base, 2 * span, 2 * span, 0); + if (q == MAP_FAILED && errno == EFAULT) + PASS(); + else + FAIL("mremap accepted two independent tracker records"); + + munmap(first, span); + munmap(second, span); + close(fd2); + close(fd1); +} + +static void test_file_backed_fork_split_move(void) +{ + TEST("MAP_SHARED file: mremap across fork-split source"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-cf-mremap-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0 || pwrite(fd, "F", 1, 0) != 1 || + pwrite(fd, "X", 1, (off_t) span) != 1) { + FAIL("file setup"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap"); + close(fd); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork"); + munmap(p, span); + close(fd); + return; + } + + if (pid == 0) { + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(40); + if (grown[0] != 'F' || grown[span] != 0) + _exit(41); + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(42); + + /* A second fork marks both tracker records inherited. Their stable + * logical-VMA lineage must still allow the grandchild to mremap the + * complete source. + */ + pid_t grandchild = fork(); + if (grandchild < 0) + _exit(43); + if (grandchild == 0) { + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved == MAP_FAILED) + _exit(44); + if (moved == grown) + _exit(45); + if (moved[0] != 'F' || moved[span] != 0 || moved[2 * span] != 0) + _exit(46); + + munmap(moved, 3 * span); + munmap(blocker, span); + _exit(0); + } + + int grandchild_status = 0; + if (waitpid(grandchild, &grandchild_status, 0) < 0 || + !WIFEXITED(grandchild_status)) + _exit(47); + if (WEXITSTATUS(grandchild_status) != 0) + _exit(WEXITSTATUS(grandchild_status)); + munmap(grown, 2 * span); + munmap(blocker, span); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid"); + } else if (!WIFEXITED(status)) { + FAIL("child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "child mremap failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_file_backed_mprotect_fragments_move(void) +{ + TEST("MAP_SHARED file: mremap across restored mprotect fragments"); + + const size_t span = 64 * 1024; + const size_t old_size = 3 * span; + const size_t new_size = 4 * span; + char tmpl[] = "/tmp/elfuse-mremap-fragments-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) new_size) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(5 * span, old_size, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'A'; + p[span] = 'B'; + p[2 * span] = 'C'; + + if (mprotect(p + span, span, PROT_READ) != 0 || + mprotect(p + span, span, PROT_READ | PROT_WRITE) != 0) { + FAIL("mprotect split and restore failed"); + munmap(p, old_size); + close(fd); + return; + } + + void *blocker = mmap(p + old_size, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != p + old_size) { + FAIL("move blocker mmap failed"); + munmap(p, old_size); + close(fd); + return; + } + + char *moved = mremap(p, old_size, new_size, MREMAP_MAYMOVE); + if (moved == MAP_FAILED || moved == p || moved[0] != 'A' || + moved[span] != 'B' || moved[2 * span] != 'C' || moved[3 * span] != 0) { + FAIL("fragmented logical VMA did not move intact"); + if (moved == MAP_FAILED) + munmap(p, old_size); + else + munmap(moved, new_size); + } else { + moved[3 * span] = 'D'; + PASS(); + munmap(moved, new_size); + } + + munmap(blocker, span); + close(fd); +} + +static void test_file_backed_fixed_move_from_same_vma(void) +{ + TEST("MAP_SHARED file: fixed subrange move keeps source fd live"); + + const size_t span = 64 * 1024; + char tmpl[] = "/tmp/elfuse-mremap-fixed-source-fd-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (3 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'D'; + p[span] = 'M'; + p[2 * span] = 'S'; + + char *moved = + mremap(p + 2 * span, span, span, MREMAP_MAYMOVE | MREMAP_FIXED, p); + if (moved != p || moved[0] != 'S' || moved[span] != 'M') + FAIL("fixed subrange move lost its source backing fd"); + else + PASS(); + + munmap(p, 3 * span); + close(fd); +} + +static void test_file_backed_mremap_emfile_atomic(void) +{ + TEST("MAP_SHARED fork growth: mremap EMFILE preserves source"); + + const size_t span = 64 * 1024; + const size_t page_size = 4096; + const int fill_limit = 4096; + char tmpl[] = "/tmp/elfuse-mremap-emfile-XXXXXX"; + int fd = mkstemp(tmpl); + if (fd < 0) { + FAIL("mkstemp failed"); + return; + } + unlink(tmpl); + if (ftruncate(fd, (off_t) (4 * span)) != 0) { + FAIL("file setup failed"); + close(fd); + return; + } + + char *p = reserve_then_map_fixed(4 * span, span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, 0); + if (p == MAP_FAILED) { + FAIL("file mmap failed"); + close(fd); + return; + } + p[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + munmap(p, span); + close(fd); + return; + } + if (pid == 0) { + const size_t filler_reservation_size = + (size_t) fill_limit * 2 * page_size; + char *grown = mremap(p, span, 2 * span, 0); + if (grown != p) + _exit(60); + grown[span] = 'T'; + grown[2 * span - 1] = 'Z'; + + void *blocker = mmap(grown + 2 * span, span, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (blocker != grown + 2 * span) + _exit(61); + + char *fixed_probe = + reserve_then_map_fixed(4 * span, 3 * span, PROT_READ | PROT_WRITE, + MAP_SHARED, fd, (off_t) span); + if (fixed_probe == MAP_FAILED) + _exit(72); + fixed_probe[0] = 'D'; + fixed_probe[2 * span] = 'S'; + + int last_dup = -1; + for (;;) { + int duplicated = dup(fd); + if (duplicated < 0) + break; + last_dup = duplicated; + } + if (errno != EMFILE || last_dup < 0) + _exit(62); + /* Keep one guest slot available for the /proc/self/maps proof below. + * The close also releases one host descriptor; the file-backed filler + * loop consumes it again before finding the actual host/table limit. */ + if (close(last_dup) != 0) + _exit(69); + + void *filler_base = mmap(NULL, filler_reservation_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (filler_base == MAP_FAILED || + munmap(filler_base, filler_reservation_size) != 0) + _exit(70); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < fill_limit; filler_count++) { + void *filler = mmap( + (char *) filler_base + (size_t) filler_count * 2 * page_size, + page_size, PROT_NONE, MAP_PRIVATE | MAP_FIXED, fd, 0); + if (filler == MAP_FAILED) + break; + last_filler = filler; + } + if (filler_count == 0 || filler_count == fill_limit || errno != ENOMEM) + _exit(63); + + /* The first failed filler proves no host descriptor remains. Freeing + * one owned tracker fd lets mremap's prefix dup succeed and forces the + * second, tail-owned dup to hit EMFILE. + */ + if (munmap(last_filler, page_size) != 0) + _exit(64); + + /* With one host and one guest descriptor available, maps must be + * readable. Near the fixed tracker capacity, VMA count alone cannot + * prove whether tracker pressure or host descriptors ended the filler + * loop, so skip instead of attributing an ambiguous ENOMEM to the + * second mremap dup. */ + int maps_count = 0; + if (!count_maps_entries(&maps_count) || maps_count >= 4000) + _exit(77); + + errno = 0; + char *moved = mremap(grown, 2 * span, 3 * span, MREMAP_MAYMOVE); + if (moved != MAP_FAILED || errno != ENOMEM) + _exit(65); + + volatile char *source = grown; + if (source[0] != 'I' || source[span] != 'T' || + source[2 * span - 1] != 'Z') + _exit(66); + if (mprotect(grown, 2 * span, PROT_READ) != 0 || source[0] != 'I' || + source[span] != 'T' || + mprotect(grown, 2 * span, PROT_READ | PROT_WRITE) != 0) + _exit(67); + source[1] = 'J'; + source[span + 1] = 'U'; + if (source[1] != 'J' || source[span + 1] != 'U') + _exit(68); + + /* Exactly one host descriptor is free again. A fixed subrange move + * consumes it for target tracking, then must fail atomically when the + * source-boundary snapshot cannot duplicate its backing fd. */ + errno = 0; + char *fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != MAP_FAILED || errno != ENOMEM || fixed_probe[0] != 'D' || + fixed_probe[2 * span] != 'S') + _exit(73); + + /* Release guest dup slots and filler tracker descriptors, then retry. + * A failed split that published backing_fd=-1 would make this retry + * fail even though descriptor capacity is now available. */ + for (int guest_fd = 0; guest_fd < 4096; guest_fd++) { + if (guest_fd != fd) + (void) close(guest_fd); + } + if (munmap(filler_base, filler_reservation_size) != 0) + _exit(74); + + fixed = mremap(fixed_probe + 2 * span, span, span, + MREMAP_MAYMOVE | MREMAP_FIXED, fixed_probe); + if (fixed != fixed_probe || fixed[0] != 'S') + _exit(75); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("EMFILE child terminated abnormally"); + } else if (WEXITSTATUS(status) == 77) { + printf("SKIP: fd exhaustion and region-table pressure are ambiguous\n"); + } else if (WEXITSTATUS(status) != 0) { + char buf[80]; + snprintf(buf, sizeof(buf), "EMFILE child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + munmap(p, span); + close(fd); +} + +static void test_heap_tail_mprotect_then_grow(void) +{ + TEST("brk growth does not reuse protected heap tail"); + + const uintptr_t page_size = 4096; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(50); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(51); + if (!set_program_break(final_end)) + _exit(52); + + if (!mapping_is_rw_private(protected_end)) + _exit(54); + + if (!set_program_break(protected_end)) + _exit(55); + if (!set_program_break(final_end)) + _exit(56); + if (!mapping_is_rw_private(protected_end)) + _exit(57); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) >= 0 && WIFEXITED(status) && + WEXITSTATUS(status) == 0) + PASS(); + else + FAIL("new brk page inherited stale tail protection"); + + (void) set_program_break(original); +} + +static void test_heap_growth_with_full_region_table(void) +{ + TEST("brk growth with full region table has no overlapping heap headers"); + + const uintptr_t page_size = 4096; + const int fill_limit = 8192; + const int minimum_fill = 2048; + void *current_break = sbrk(0); + if (current_break == (void *) -1) { + FAIL("read parent brk failed"); + return; + } + uintptr_t original = (uintptr_t) current_break; + uintptr_t inherited_end = + (original + page_size - 1) / page_size * page_size + page_size; + if (!set_program_break(inherited_end)) { + FAIL("parent brk growth failed"); + return; + } + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) set_program_break(original); + return; + } + if (pid == 0) { + uintptr_t protected_end = inherited_end + page_size; + uintptr_t final_end = protected_end + page_size; + if (!set_program_break(protected_end)) + _exit(80); + if (mprotect((void *) inherited_end, page_size, PROT_READ) != 0) + _exit(81); + + int filled = 0; + for (; filled < fill_limit; filled++) { + int prot = (filled & 1) ? PROT_NONE : PROT_READ; + void *q = + mmap(NULL, page_size, prot, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (q == MAP_FAILED) + break; + } + if (filled < minimum_fill || filled == fill_limit || errno != ENOMEM) + _exit(82); + + if (!set_program_break(final_end)) + _exit(83); + volatile char *new_page = (char *) protected_end; + new_page[0] = 'H'; + if (new_page[0] != 'H') + _exit(84); + + int heap_count = 0; + if (!heap_headers_do_not_overlap(&heap_count) || heap_count < 2) + _exit(85); + _exit(0); + } + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("waitpid failed"); + } else if (!WIFEXITED(status)) { + FAIL("region-table child terminated abnormally"); + } else if (WEXITSTATUS(status) != 0) { + char buf[96]; + snprintf(buf, sizeof(buf), "region-table child failed at step %d", + WEXITSTATUS(status)); + FAIL(buf); + } else { + PASS(); + } + + (void) set_program_break(original); +} + +int main(void) +{ + printf("test-mremap-fork-tracking: elfuse mremap tracker tests\n"); + + test_postfork_adjacent_anon_rejected(); + test_adjacent_file_vmas_rejected(); + test_file_backed_fork_split_move(); + test_file_backed_mprotect_fragments_move(); + test_file_backed_fixed_move_from_same_vma(); + test_file_backed_mremap_emfile_atomic(); + test_heap_tail_mprotect_then_grow(); + test_heap_growth_with_full_region_table(); + + SUMMARY("test-mremap-fork-tracking"); + return fails > 0 ? 1 : 0; +} diff --git a/tests/test-mremap-tail-emfile.c b/tests/test-mremap-tail-emfile.c new file mode 100644 index 00000000..ce94e872 --- /dev/null +++ b/tests/test-mremap-tail-emfile.c @@ -0,0 +1,235 @@ +/* + * test-mremap-tail-emfile exercises file-backed mremap bookkeeping while the + * guest and host descriptor tables are exhausted. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * This probe deliberately uses libc interfaces so it can serve as the + * repository's portable C implementation of the regression. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +#ifndef MREMAP_MAYMOVE +#define MREMAP_MAYMOVE 1 +#endif + +static const size_t PAGE_SIZE = 4096; +static const size_t SPAN = 64 * 1024; +static const int FILL_LIMIT = 4096; + +static void child_exit(int status) +{ + _exit(status); +} + +static void child_fail(int status) +{ + child_exit(status); +} + +static void child_probe(int file_fd, unsigned char *base) +{ + void *result = mremap(base, SPAN, 3 * SPAN, 0); + if (result == MAP_FAILED || result != base) + child_fail(10); + + base[SPAN] = 'T'; + base[2 * SPAN] = 'R'; + base[SPAN + PAGE_SIZE] = 'S'; + base[SPAN + 3 * PAGE_SIZE] = 'M'; + + /* Exhaust both descriptor tables, then leave one slot available. */ + int last_dup = -1; + int dup_error = 0; + for (;;) { + int duplicated = dup(file_fd); + if (duplicated < 0) { + dup_error = errno; + break; + } + last_dup = duplicated; + } + if (dup_error != EMFILE || last_dup < 0) + child_fail(11); + if (close(last_dup) != 0) + child_fail(12); + + /* Fill the region table with one-page file mappings separated by holes. */ + const size_t filler_length = (size_t) FILL_LIMIT * 2 * PAGE_SIZE; + void *reserved = mmap(NULL, filler_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) + child_fail(13); + if (munmap(reserved, filler_length) != 0) + child_fail(13); + + void *last_filler = MAP_FAILED; + int filler_count = 0; + for (; filler_count < FILL_LIMIT; filler_count++) { + void *address = + (char *) reserved + (size_t) filler_count * 2 * PAGE_SIZE; + void *filler = mmap(address, PAGE_SIZE, PROT_NONE, + MAP_PRIVATE | MAP_FIXED, file_fd, 0); + if (filler == MAP_FAILED) { + int filler_errno = errno; + if (filler_errno != ENOMEM || filler_count == 0) + child_fail(filler_errno == ENOMEM ? 16 : 14); + break; + } + if (filler != address) + child_fail(15); + last_filler = filler; + } + if (filler_count == FILL_LIMIT || last_filler == MAP_FAILED) + child_fail(16); + if (munmap(last_filler, PAGE_SIZE) != 0) + child_fail(17); + + /* Consume the one descriptor released by the last filler mapping. */ + int probe_fd = dup(file_fd); + if (probe_fd < 0) + child_fail(22); + + /* Splitting either boundary of the child-private tail must fail before + * changing memory or PTEs when no descriptor is available. */ + errno = 0; + result = mremap(base, SPAN + 2 * PAGE_SIZE, SPAN + PAGE_SIZE, 0); + int remap_errno = errno; + if (result != MAP_FAILED || remap_errno != ENOMEM || + base[SPAN + PAGE_SIZE] != 'S') + child_fail(23); + + errno = 0; + int munmap_result = munmap(base + SPAN + 3 * PAGE_SIZE, PAGE_SIZE); + int munmap_errno = errno; + if (munmap_result != -1 || munmap_errno != ENOMEM || + base[SPAN + 3 * PAGE_SIZE] != 'M') + child_fail(25); + + if (close(probe_fd) != 0) + child_fail(27); + + /* old_size ends inside the child-private tail. */ + errno = 0; + result = + mremap(base, SPAN + PAGE_SIZE, 2 * SPAN + PAGE_SIZE, MREMAP_MAYMOVE); + remap_errno = errno; + + /* Drop all duplicate guest slots, retaining the original file fd. */ + for (int fd = 0; fd < FILL_LIMIT; fd++) { + if (fd != file_fd) + (void) close(fd); + } + if (munmap(reserved, filler_length) != 0) + child_fail(18); + + /* A successful move owns a target mapping; release it before msync. */ + if (result != MAP_FAILED) { + if (munmap(result, 2 * SPAN + PAGE_SIZE) != 0) + child_fail(21); + } else if (remap_errno != ENOMEM) { + child_fail(21); + } + + if (msync(base + 2 * SPAN, SPAN, MS_SYNC) != 0) + child_fail(19); + + unsigned char byte = 0; + if (pread(file_fd, &byte, 1, 2 * SPAN) != 1 || byte != 'R') + child_fail(20); + + child_exit(0); +} + +static void test_file_backed_mremap_tail_emfile(void) +{ + TEST("MAP_SHARED mremap tail EMFILE preserves source"); + + char path[] = "/tmp/elfuse-mremap-tail-emfile-XXXXXX"; + int file_fd = mkstemp(path); + if (file_fd < 0) { + FAIL("create temp file failed"); + return; + } + (void) unlink(path); + + const size_t file_length = 4 * SPAN; + if (ftruncate(file_fd, (off_t) file_length) != 0) { + FAIL("truncate temp file failed"); + close(file_fd); + return; + } + + const size_t reservation_length = file_length + PAGE_SIZE; + void *reserved = mmap(NULL, reservation_length, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED || munmap(reserved, reservation_length) != 0) { + FAIL("reserve source range failed"); + close(file_fd); + return; + } + + void *source_address = (char *) reserved + PAGE_SIZE; + void *mapped = mmap(source_address, SPAN, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, file_fd, 0); + if (mapped == MAP_FAILED || mapped != source_address) { + FAIL("map source file failed"); + close(file_fd); + return; + } + + unsigned char *base = mapped; + base[0] = 'I'; + + pid_t pid = fork(); + if (pid < 0) { + FAIL("fork failed"); + (void) munmap(base, SPAN); + close(file_fd); + return; + } + if (pid == 0) + child_probe(file_fd, base); + + int status = 0; + if (waitpid(pid, &status, 0) < 0) { + FAIL("wait failed"); + } else if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + char message[96]; + if (WIFEXITED(status)) + snprintf(message, sizeof(message), "child failed at step %d", + WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) + snprintf(message, sizeof(message), "child killed by signal %d", + WTERMSIG(status)); + else + snprintf(message, sizeof(message), "child ended unexpectedly"); + FAIL(message); + } else { + PASS(); + } + + (void) munmap(base, SPAN); + close(file_fd); +} + +int main(void) +{ + printf("test-mremap-tail-emfile: file-backed mremap atomicity\n"); + test_file_backed_mremap_tail_emfile(); + SUMMARY("test-mremap-tail-emfile"); + return fails != 0; +} diff --git a/tests/test-proc-smap.c b/tests/test-proc-smap.c new file mode 100644 index 00000000..5be4802d --- /dev/null +++ b/tests/test-proc-smap.c @@ -0,0 +1,675 @@ +/* + * Generic /proc//smaps parser and accounting regression test. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "test-harness.h" + +int passes = 0, fails = 0; + +typedef struct { + uintptr_t start; + uintptr_t end; + unsigned long long offset; + char perms[5]; + unsigned long long size_kb; + unsigned long long kernel_page_kb; + unsigned long long mmu_page_kb; + unsigned long long shared_dirty_kb; + bool have_size; + bool have_shared_dirty; + bool have_vmflags; + bool vmflags_wr; +} smaps_vma_t; + +static const char *const smaps_fields[] = { + "Size:", "KernelPageSize:", "MMUPageSize:", "Rss:", + "Pss:", "Pss_Dirty:", "Shared_Clean:", "Shared_Dirty:", + "Private_Clean:", "Private_Dirty:", "Referenced:", "Anonymous:", + "KSM:", "LazyFree:", "AnonHugePages:", "ShmemPmdMapped:", + "FilePmdMapped:", "Shared_Hugetlb:", "Private_Hugetlb:", "Swap:", + "SwapPss:", "Locked:", "THPeligible:", "ProtectionKey:", + "VmFlags:", +}; + +#define SMAPS_FIELD_COUNT (sizeof(smaps_fields) / sizeof(smaps_fields[0])) +#define SMAPS_KB_FIELD_COUNT 22 + +typedef struct { + smaps_vma_t *vmas; + size_t count; +} smaps_info_t; + +static const char *skip_space(const char *p) +{ + while (*p && isspace((unsigned char) *p)) + p++; + return p; +} + +static bool parse_token(const char **cursor, char *token, size_t token_size) +{ + const char *p = skip_space(*cursor); + const char *start = p; + while (*p && !isspace((unsigned char) *p)) + p++; + size_t len = (size_t) (p - start); + if (len == 0 || len >= token_size) + return false; + memcpy(token, start, len); + token[len] = '\0'; + *cursor = p; + return true; +} + +static bool parse_unsigned_token(const char *token, + int base, + unsigned long long *value) +{ + if (!token[0]) + return false; + for (const unsigned char *p = (const unsigned char *) token; *p; p++) { + if (base == 16 ? !isxdigit(*p) : !isdigit(*p)) + return false; + } + + errno = 0; + char *end = NULL; + unsigned long long n = strtoull(token, &end, base); + if (errno == ERANGE || end == token || *end != '\0') + return false; + *value = n; + return true; +} + +static bool parse_device_token(const char *token) +{ + const char *colon = strchr(token, ':'); + if (!colon || colon == token || colon[1] == '\0' || strchr(colon + 1, ':')) + return false; + + char major[32], minor[32]; + size_t major_len = (size_t) (colon - token); + if (major_len >= sizeof(major) || strlen(colon + 1) >= sizeof(minor)) + return false; + memcpy(major, token, major_len); + major[major_len] = '\0'; + strcpy(minor, colon + 1); + + unsigned long long value; + return parse_unsigned_token(major, 16, &value) && + parse_unsigned_token(minor, 16, &value); +} + +/* Parse the address/perms/offset/dev/inode part of a smaps header. The rest + * of the line is an optional pathname and is intentionally left opaque. */ +static bool parse_header(const char *line, smaps_vma_t *vma) +{ + const char *p = line; + errno = 0; + char *end = NULL; + unsigned long long start = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || *end != '-') + return false; + p = end + 1; + + errno = 0; + unsigned long long finish = strtoull(p, &end, 16); + if (errno == ERANGE || end == p || start >= finish || + !isspace((unsigned char) *end)) + return false; + if (start > UINTPTR_MAX || finish > UINTPTR_MAX) + return false; + p = end; + + char token[128]; + if (!parse_token(&p, token, sizeof(token)) || strlen(token) != 4) + return false; + for (size_t i = 0; i < 3; i++) { + if (token[i] != 'r' && token[i] != 'w' && token[i] != 'x' && + token[i] != '-') + return false; + } + if (token[3] != 'p' && token[3] != 's') + return false; + memcpy(vma->perms, token, sizeof(vma->perms)); + + if (!parse_token(&p, token, sizeof(token)) || + !parse_unsigned_token(token, 16, &vma->offset)) + return false; + if (!parse_token(&p, token, sizeof(token)) || !parse_device_token(token)) + return false; + if (!parse_token(&p, token, sizeof(token))) + return false; + unsigned long long inode; + if (!parse_unsigned_token(token, 10, &inode)) + return false; + + vma->start = (uintptr_t) start; + vma->end = (uintptr_t) finish; + vma->size_kb = 0; + vma->kernel_page_kb = 0; + vma->mmu_page_kb = 0; + vma->shared_dirty_kb = 0; + vma->have_size = false; + vma->have_shared_dirty = false; + vma->have_vmflags = false; + vma->vmflags_wr = false; + return true; +} + +/* Parse a decimal field and optionally require a suffix after its number. + * Both smaps field families share the same label/whitespace/overflow rules; + * only the trailing unit differs ("kB" for the first group, none for the + * remaining numeric fields). */ +static bool parse_numeric_field(const char *line, + const char *label, + const char *suffix, + unsigned long long *value) +{ + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + const char *p = skip_space(line + label_len); + const char *start = p; + while (isdigit((unsigned char) *p)) + p++; + if (p == start) + return false; + char number[64]; + size_t number_len = (size_t) (p - start); + if (number_len >= sizeof(number)) + return false; + memcpy(number, start, number_len); + number[number_len] = '\0'; + if (!parse_unsigned_token(number, 10, value)) + return false; + p = skip_space(p); + return suffix ? !strcmp(p, suffix) : *p == '\0'; +} + +static bool parse_vmflags(const char *line, bool *writable) +{ + const char *label = "VmFlags:"; + size_t label_len = strlen(label); + if (strncmp(line, label, label_len) != 0) + return false; + + const char *p = line + label_len; + bool has_wr = false; + while (*(p = skip_space(p))) { + const char *start = p; + while (*p && !isspace((unsigned char) *p)) { + if (!isalpha((unsigned char) *p)) + return false; + p++; + } + if (p == start) + return false; + if ((size_t) (p - start) == 2 && start[0] == 'w' && start[1] == 'r') + has_wr = true; + } + /* A synthetic PROT_NONE VMA has no provable access flags and therefore + * may legitimately emit an empty VmFlags field. */ + *writable = has_wr; + return true; +} + +static bool finish_vma(smaps_vma_t *vma, size_t field_index) +{ + return field_index == SMAPS_FIELD_COUNT && vma->have_size && + vma->have_shared_dirty && vma->have_vmflags; +} + +/* THPeligible and ProtectionKey are conditional in real Linux kernels. The + * elfuse provider always emits both, but qemu may legitimately omit either; + * when the parser reaches one of those labels, advance over any missing + * optional fields before parsing VmFlags. */ +static void skip_optional_fields(const char *line, size_t *field_index) +{ + if (*field_index == SMAPS_FIELD_COUNT - 3 && + (!strncmp(line, "ProtectionKey:", strlen("ProtectionKey:")) || + !strncmp(line, "VmFlags:", 8))) + (*field_index)++; + if (*field_index == SMAPS_FIELD_COUNT - 2 && !strncmp(line, "VmFlags:", 8)) + (*field_index)++; +} + +static bool append_vma(smaps_info_t *info, const smaps_vma_t *vma) +{ + smaps_vma_t *vmas = + realloc(info->vmas, (info->count + 1) * sizeof(*info->vmas)); + if (!vmas) + return false; + vmas[info->count++] = *vma; + info->vmas = vmas; + return true; +} + +static bool parse_smaps(char *buf, size_t len, smaps_info_t *info) +{ + memset(info, 0, sizeof(*info)); + if (len == 0 || buf[len - 1] != '\n') + return false; /* catches a truncated final record */ + + smaps_vma_t current; + bool have_current = false; + size_t field_index = 0; + uintptr_t previous_end = 0; + char *line = buf; + while ((size_t) (line - buf) < len) { + char *next = memchr(line, '\n', len - (size_t) (line - buf)); + if (!next) + goto fail; + *next = '\0'; + + /* Linux normally places headers back-to-back; the synthetic proc + * provider separates records with one blank line. Accept that + * separator only after a complete record. */ + if (!*line) { + if (!have_current || field_index != SMAPS_FIELD_COUNT) + goto fail; + line = next + 1; + continue; + } + + smaps_vma_t header; + if (parse_header(line, &header)) { + if (have_current) { + if (!finish_vma(¤t, field_index) || + !append_vma(info, ¤t)) + goto fail; + } + if (info->count > 0 && + (header.start < previous_end || + header.start <= info->vmas[info->count - 1].start)) + goto fail; + current = header; + have_current = true; + field_index = 0; + previous_end = header.end; + } else { + if (!have_current) + goto fail; + skip_optional_fields(line, &field_index); + if (field_index >= SMAPS_FIELD_COUNT) + goto fail; + unsigned long long value; + bool writable; + if (field_index < SMAPS_KB_FIELD_COUNT && + parse_numeric_field(line, smaps_fields[field_index], "kB", + &value)) { + if (field_index == 0) + current.size_kb = value; + if (field_index == 1) + current.kernel_page_kb = value; + if (field_index == 2) + current.mmu_page_kb = value; + if (field_index == 7) + current.shared_dirty_kb = value; + if (field_index == 0) + current.have_size = true; + if (field_index == 7) + current.have_shared_dirty = true; + field_index++; + } else if (field_index >= SMAPS_KB_FIELD_COUNT && + field_index < SMAPS_FIELD_COUNT - 1 && + parse_numeric_field(line, smaps_fields[field_index], + NULL, &value)) { + field_index++; + } else if (field_index == SMAPS_FIELD_COUNT - 1 && + parse_vmflags(line, &writable)) { + current.have_vmflags = true; + current.vmflags_wr = writable; + field_index++; + } else { + goto fail; + } + } + line = next + 1; + } + + if (!have_current || !finish_vma(¤t, field_index) || + !append_vma(info, ¤t) || info->count == 0) + goto fail; + return true; + +fail: + free(info->vmas); + memset(info, 0, sizeof(*info)); + return false; +} + +static void free_smaps(smaps_info_t *info) +{ + free(info->vmas); + memset(info, 0, sizeof(*info)); +} + +/* Read, parse, and release one smaps snapshot. The parser owns only its VMA + * array, so the transient file buffer can be cleaned up in this single place + * on both success and failure. */ +static bool load_smaps(const char *path, smaps_info_t *info) +{ + FILE *file = fopen(path, "r"); + if (!file) + return false; + + char *buf = NULL; + size_t capacity = 0; + ssize_t length = getdelim(&buf, &capacity, '\0', file); + bool ok = length >= 0 && parse_smaps(buf, (size_t) length, info); + free(buf); + (void) fclose(file); + return ok; +} + +static const smaps_vma_t *find_vma(const smaps_info_t *info, uintptr_t address) +{ + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].start <= address && address < info->vmas[i].end) + return &info->vmas[i]; + } + return NULL; +} + +static size_t count_vmas_in_range(const smaps_info_t *info, + uintptr_t start, + uintptr_t end) +{ + size_t count = 0; + for (size_t i = 0; i < info->count; i++) { + if (info->vmas[i].end > start && info->vmas[i].start < end) + count++; + } + return count; +} + +static bool validate_layout(const smaps_info_t *info, + uintptr_t target, + uintptr_t stress, + size_t page_size, + size_t stress_size) +{ + if (info->count <= 256) + return false; + + const smaps_vma_t *first = find_vma(info, target); + const smaps_vma_t *middle = find_vma(info, target + page_size); + const smaps_vma_t *last = find_vma(info, target + 2 * page_size); + unsigned long long page_kb = page_size / 1024; + if (!first || !middle || !last || page_kb == 0) + return false; + + if (first->start != target || first->end != target + page_size || + middle->start != target + page_size || + middle->end != target + 2 * page_size || + last->start != target + 2 * page_size || + last->end != target + 3 * page_size) + return false; + if (strcmp(first->perms, "rw-p") || strcmp(middle->perms, "r--p") || + strcmp(last->perms, "rw-p")) + return false; + if (first->size_kb != page_kb || middle->size_kb != page_kb || + last->size_kb != page_kb || !first->vmflags_wr || middle->vmflags_wr || + !last->vmflags_wr || first->kernel_page_kb != 4 || + first->mmu_page_kb != 4 || middle->kernel_page_kb != 4 || + middle->mmu_page_kb != 4 || last->kernel_page_kb != 4 || + last->mmu_page_kb != 4) + return false; + + /* Every stress-map page alternates permissions, so each page must remain + * its own VMA. Requiring the full count makes a short read or a producer + * cap observable instead of merely checking that some blocks exceed 256. + */ + size_t expected_stress_vmas = stress_size / page_size; + if (expected_stress_vmas <= 256 || + count_vmas_in_range(info, stress, stress + stress_size) != + expected_stress_vmas) + return false; + for (size_t i = 0; i < expected_stress_vmas; i++) { + const smaps_vma_t *page = find_vma(info, stress + i * page_size); + if (!page || page->start != stress + i * page_size || + page->end != stress + (i + 1) * page_size) + return false; + } + return true; +} + +static bool read_exact(int fd, void *data, size_t len) +{ + char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = read(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + +static bool write_exact(int fd, const void *data, size_t len) +{ + const char *p = data; + size_t done = 0; + while (done < len) { + ssize_t n = write(fd, p + done, len - done); + if (n < 0 && errno == EINTR) + continue; + if (n <= 0) + return false; + done += (size_t) n; + } + return true; +} + +static int child_probe(uintptr_t target, + uintptr_t stress, + size_t page_size, + size_t stress_size) +{ + void *postfork = MAP_FAILED; + uintptr_t target_end = target + 3 * page_size; + uintptr_t stress_end = stress + stress_size; + +#ifdef MAP_FIXED_NOREPLACE + /* Keep the probe in a known gap so the synthetic smaps builder cannot + * merge it with the target's final rw page or the stress fixture. */ + uintptr_t fixed_hint = 0x700000000000ULL; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) fixed_hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + if (candidate != MAP_FAILED) { + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + } + fixed_hint += 16 * page_size; + } +#endif + + if (postfork == MAP_FAILED) { + uintptr_t hint = stress_end + 64 * page_size; + for (int attempt = 0; attempt < 32; attempt++) { + void *candidate = + mmap((void *) hint, page_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (candidate == MAP_FAILED) + break; + uintptr_t p = (uintptr_t) candidate; + if (p != target_end && p + page_size != target && p != stress_end && + p + page_size != stress) { + postfork = candidate; + break; + } + munmap(candidate, page_size); + hint += 64 * page_size; + } + } + if (postfork == MAP_FAILED) + return 1; + + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + if (!load_smaps(paths[i], &info)) { + munmap(postfork, page_size); + return 1; + } + bool ok = + validate_layout(&info, target, stress, page_size, stress_size); + const smaps_vma_t *first = find_vma(&info, target); + const smaps_vma_t *middle = find_vma(&info, target + page_size); + const smaps_vma_t *last = find_vma(&info, target + 2 * page_size); + const smaps_vma_t *stress_ro = find_vma(&info, stress); + const smaps_vma_t *stress_rw = find_vma(&info, stress + page_size); + const smaps_vma_t *postfork_vma = find_vma(&info, (uintptr_t) postfork); + if (!ok || !first || !middle || !last || !stress_ro || !stress_rw || + !postfork_vma || first->shared_dirty_kb == 0 || + middle->shared_dirty_kb != 0 || last->shared_dirty_kb == 0 || + stress_ro->shared_dirty_kb != 0 || + stress_rw->shared_dirty_kb == 0 || + postfork_vma->shared_dirty_kb != 0) { + free_smaps(&info); + munmap(postfork, page_size); + return 1; + } + free_smaps(&info); + } + munmap(postfork, page_size); + return 0; +} + +int main(void) +{ + long page_size_long = sysconf(_SC_PAGESIZE); + TEST("smaps page size and VMA fixture"); + if (page_size_long < 1024 || page_size_long % 1024 != 0) { + FAIL("sysconf(_SC_PAGESIZE)"); + SUMMARY("test-proc-smap"); + return 1; + } + PASS(); + size_t page_size = (size_t) page_size_long; + + size_t target_size = 3 * page_size; + char *target = mmap(NULL, target_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + size_t stress_pages = 320; + size_t stress_size = stress_pages * page_size; + char *stress = MAP_FAILED; + bool fixture_ok = target != MAP_FAILED; + if (fixture_ok) + stress = mmap(NULL, stress_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + fixture_ok = fixture_ok && stress != MAP_FAILED; + if (fixture_ok) { + *(volatile unsigned char *) target = 0x5a; /* make one page dirty */ + fixture_ok = mprotect(target + page_size, page_size, PROT_READ) == 0; + } + if (fixture_ok) { + /* Start with a read-only page so an allocator placing this mapping + * immediately after target cannot merge target's final rw page into + * the stress range. Alternating permissions keeps every stress page + * as a distinct VMA while preserving the >256 completeness probe. */ + for (size_t i = 0; i < stress_pages; i += 2) { + if (mprotect(stress + i * page_size, page_size, PROT_READ) < 0) { + fixture_ok = false; + break; + } + } + } + + TEST("smaps headers, order, fields, and completeness"); + if (!fixture_ok) { + FAIL("mmap/mprotect fixture"); + } else { + char pid_path[64]; + snprintf(pid_path, sizeof(pid_path), "/proc/%ld/smaps", + (long) getpid()); + const char *paths[] = {"/proc/self/smaps", pid_path}; + bool ok = true; + size_t expected_count = 0; + for (size_t i = 0; i < sizeof(paths) / sizeof(paths[0]); i++) { + smaps_info_t info; + bool parsed = load_smaps(paths[i], &info); + if (!parsed || + !validate_layout(&info, (uintptr_t) target, (uintptr_t) stress, + page_size, stress_size)) { + ok = false; + } else if (i == 0) { + expected_count = info.count; + } else if (info.count != expected_count) { + ok = false; + } + if (parsed) + free_smaps(&info); + } + EXPECT_TRUE(ok, "smaps parser/layout/completeness"); + } + + TEST("fork Shared_Dirty inheritance and post-fork exclusion"); + if (!fixture_ok) { + FAIL("fixture unavailable"); + } else { + int pipefd[2] = {-1, -1}; + pid_t pid = pipe(pipefd) == 0 ? fork() : -1; + if (pid < 0) { + FAIL("pipe/fork"); + if (pipefd[0] >= 0) + close(pipefd[0]); + if (pipefd[1] >= 0) + close(pipefd[1]); + } else if (pid == 0) { + close(pipefd[0]); + int result = child_probe((uintptr_t) target, (uintptr_t) stress, + page_size, stress_size); + (void) write_exact(pipefd[1], &result, sizeof(result)); + close(pipefd[1]); + _exit(result); + } else { + close(pipefd[1]); + int result = 1; + bool received = read_exact(pipefd[0], &result, sizeof(result)); + close(pipefd[0]); + int status = 0; + bool waited = waitpid(pid, &status, 0) == pid; + EXPECT_TRUE(received && waited && result == 0 && + WIFEXITED(status) && WEXITSTATUS(status) == 0, + "fork Shared_Dirty accounting"); + } + } + + if (stress != MAP_FAILED) + munmap(stress, stress_size); + if (target != MAP_FAILED) + munmap(target, target_size); + SUMMARY("test-proc-smap"); + return fails == 0 ? 0 : 1; +} diff --git a/tests/test-string-builder-host.c b/tests/test-string-builder-host.c new file mode 100644 index 00000000..e404680c --- /dev/null +++ b/tests/test-string-builder-host.c @@ -0,0 +1,230 @@ +/* + * Native-host unit tests for string_builder_t. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include "string-builder.h" + +static void expect_string(const string_builder_t *builder, const char *expected) +{ + size_t length = strlen(expected); + assert(string_builder_length(builder) == length); + assert(builder->storage.raw.count == length); + if (string_builder_data_const(builder) != NULL) { + assert(strcmp(string_builder_data_const(builder), expected) == 0); + assert(string_builder_data_const(builder)[length] == '\0'); + } else { + assert(length == 0); + } +} + +static void test_zero_and_initial_capacity(void) +{ + /* Fresh automatic objects need not be manually zeroed before init. */ + string_builder_t fresh; + assert(string_builder_init(&fresh, 0) == 0); + assert(string_builder_append(&fresh, "fresh") == 0); + expect_string(&fresh, "fresh"); + string_builder_destroy(&fresh); + + string_builder_t fresh_capacity; + assert(string_builder_init(&fresh_capacity, 32) == 0); + assert(string_builder_capacity(&fresh_capacity) >= 32); + string_builder_destroy(&fresh_capacity); + + string_builder_t zero = {0}; + assert(string_builder_data(&zero) == NULL); + assert(string_builder_length(&zero) == 0); + assert(string_builder_capacity(&zero) == 0); + assert(string_builder_appendf(&zero, "%c", '\0') == 0); + assert(string_builder_data(&zero) == NULL); + /* The public API also accepts a plain {0} value without an init call. */ + assert(string_builder_append(&zero, "zero") == 0); + expect_string(&zero, "zero"); + string_builder_destroy(&zero); + + string_builder_t lazy = {0}; + assert(string_builder_init(&lazy, 0) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_capacity(&lazy) == 0); + assert(string_builder_data(&lazy) == NULL); + assert(string_builder_reserve(&lazy, 4) == 0); + assert(string_builder_length(&lazy) == 0); + assert(string_builder_data(&lazy)[0] == '\0'); + assert(string_builder_append(&lazy, "lazy") == 0); + assert(string_builder_capacity(&lazy) >= string_builder_length(&lazy) + 1); + expect_string(&lazy, "lazy"); + string_builder_destroy(&lazy); + + string_builder_t initial = {0}; + /* initial_capacity includes the trailing NUL byte. */ + assert(string_builder_init(&initial, 32) == 0); + assert(string_builder_capacity(&initial) >= 32); + assert(string_builder_data(&initial) != NULL); + assert(string_builder_length(&initial) == 0); + expect_string(&initial, ""); + string_builder_destroy(&initial); +} + +static void test_text_and_formatted_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + assert(string_builder_append(&builder, "prefix") == 0); + + assert(string_builder_appendf(&builder, ":%s:%d", "formatted", 42) == 0); + expect_string(&builder, "prefix:formatted:42"); + + /* An empty C string is a no-op. */ + size_t old_length = string_builder_length(&builder); + assert(string_builder_append(&builder, "") == 0); + assert(string_builder_length(&builder) == old_length); + expect_string(&builder, "prefix:formatted:42"); + string_builder_destroy(&builder); +} + +static void test_c_string_semantics(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "prefix") == 0); + + const char embedded[] = {'a', '\0', 'b', '\0'}; + errno = 0; + assert(string_builder_append(&builder, embedded) == 0); + assert(errno == 0); + expect_string(&builder, "prefixa"); + + errno = 0; + assert(string_builder_appendf(&builder, "x%c y", '\0') == 0); + assert(errno == 0); + expect_string(&builder, "prefixax"); + string_builder_destroy(&builder); + + /* A formatted NUL also terminates the appended C-string prefix when the + * first sizing pass has room. + */ + string_builder_t fit = {0}; + assert(string_builder_init(&fit, 16) == 0); + errno = 0; + assert(string_builder_appendf(&fit, "a%c%d", '\0', 1) == 0); + assert(errno == 0); + expect_string(&fit, "a"); + string_builder_destroy(&fit); +} + +static void test_formatted_append_reserves_terminator(void) +{ + string_builder_t builder = {0}; + + /* A formatted append must reserve one byte beyond the visible payload for + * the builder's trailing NUL. This exact-length payload used to make the + * generic array allocate only the payload bytes before the terminator was + * written. + */ + assert(string_builder_appendf(&builder, "%s", "12345678") == 0); + expect_string(&builder, "12345678"); + assert(string_builder_capacity(&builder) >= + string_builder_length(&builder) + 1); + string_builder_destroy(&builder); +} + +static void test_growth_preserves_content(void) +{ + enum { COUNT = 4096 }; + char expected[COUNT]; + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 1) == 0); + + for (size_t i = 0; i < COUNT; i++) { + expected[i] = (char) ('A' + (i % 26)); + char chunk[2] = {expected[i], '\0'}; + assert(string_builder_append(&builder, chunk) == 0); + } + assert(string_builder_length(&builder) == COUNT); + assert(memcmp(string_builder_data_const(&builder), expected, COUNT) == 0); + assert(string_builder_data_const(&builder)[COUNT] == '\0'); + assert(string_builder_capacity(&builder) >= COUNT + 1); + string_builder_destroy(&builder); +} + +static void test_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 4) == 0); + assert(string_builder_append(&builder, "abc") == 0); + const char *alias = string_builder_data_const(&builder) + 1; + assert(string_builder_append(&builder, alias) == 0); + const char expected[] = "abcbc"; + expect_string(&builder, expected); + string_builder_destroy(&builder); +} + +static void test_formatted_alias_append(void) +{ + string_builder_t builder = {0}; + assert(string_builder_append(&builder, "x%s") == 0); + const char *alias = string_builder_data_const(&builder); +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wformat-nonliteral" + assert(string_builder_appendf(&builder, alias, alias) == 0); +#pragma clang diagnostic pop + expect_string(&builder, "x%sxx%s"); + string_builder_destroy(&builder); +} + +static void test_overflow_preserves_content(void) +{ + string_builder_t builder = {0}; + assert(string_builder_init(&builder, 0) == 0); + assert(string_builder_appendf(&builder, "prefix:%d", 7) == 0); + + char snapshot[32]; + assert(string_builder_length(&builder) < sizeof(snapshot)); + memcpy(snapshot, string_builder_data_const(&builder), + string_builder_length(&builder)); + size_t old_length = string_builder_length(&builder); + size_t old_capacity = string_builder_capacity(&builder); + + errno = 0; + assert(string_builder_reserve(&builder, SIZE_MAX) == -1); + assert(errno == EOVERFLOW); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + + errno = 0; + assert(string_builder_append(&builder, NULL) == -1); + assert(errno == EILSEQ); + assert(string_builder_length(&builder) == old_length); + assert(string_builder_capacity(&builder) == old_capacity); + assert(memcmp(string_builder_data_const(&builder), snapshot, old_length) == + 0); + assert(string_builder_data_const(&builder)[old_length] == '\0'); + string_builder_destroy(&builder); +} + +int main(void) +{ + test_zero_and_initial_capacity(); + test_text_and_formatted_append(); + test_c_string_semantics(); + test_formatted_append_reserves_terminator(); + test_growth_preserves_content(); + test_alias_append(); + test_formatted_alias_append(); + test_overflow_preserves_content(); + puts("test-string-builder-host: PASS"); + return 0; +} diff --git a/tests/test-util.h b/tests/test-util.h index 5a1a1a71..a8b6c146 100644 --- a/tests/test-util.h +++ b/tests/test-util.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,70 @@ static inline ssize_t read_file_nul(const char *path, char *buf, size_t bufsz) return total; } +/* Read a file whose size is not available from st_size (for example a proc + * file) until EOF, growing the buffer and appending a NUL terminator. */ +static inline ssize_t read_file_dynamic_nul(const char *path, + char **buf_out, + size_t *len_out) +{ + if (!path || !buf_out || !len_out) { + errno = EINVAL; + return -1; + } + + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + + size_t cap = 64 * 1024; + size_t len = 0; + char *buf = malloc(cap); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + + for (;;) { + if (len + 1 >= cap) { + if (cap > SIZE_MAX / 2) { + free(buf); + close(fd); + errno = EOVERFLOW; + return -1; + } + size_t new_cap = cap * 2; + char *new_buf = realloc(buf, new_cap); + if (!new_buf) { + free(buf); + close(fd); + errno = ENOMEM; + return -1; + } + buf = new_buf; + cap = new_cap; + } + + ssize_t n = read(fd, buf + len, cap - len - 1); + if (n < 0 && errno == EINTR) + continue; + if (n < 0) { + free(buf); + close(fd); + return -1; + } + if (n == 0) + break; + len += (size_t) n; + } + close(fd); + + buf[len] = '\0'; + *buf_out = buf; + *len_out = len; + return (ssize_t) len; +} + static inline ssize_t raw_read_fd_all_nul(int fd, char *buf, size_t bufsz) { if (bufsz == 0)