diff --git a/BUILD.bazel b/BUILD.bazel index 0d734ad4f..a80f905e9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,6 +6,7 @@ filegroup( [ "src/snmalloc/**/*", "src/test/*.h", + "src/test/*.cc", "CMakeLists.txt", ], ), diff --git a/CMakeLists.txt b/CMakeLists.txt index e017d1e7a..7e8edcfe8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -449,6 +449,11 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) target_compile_definitions(${TESTNAME} PRIVATE ${DEFINES}) endif() + # Link the pre-compiled test library into all tests. For tests + # that include snmalloc.h directly the linker will simply discard + # the duplicate inline definitions (ODR-safe). + target_link_libraries(${TESTNAME} snmalloc-testlib-${TAG}) + if (${TEST} MATCHES "release-.*") message(VERBOSE "Adding test: ${TESTNAME} only for release configs") add_test(NAME ${TESTNAME} COMMAND ${TESTNAME} CONFIGURATIONS "Release") @@ -480,6 +485,18 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) endforeach() endfunction() + function(build_test_library TAG DEFINES) + set(LIBNAME snmalloc-testlib-${TAG}) + add_library(${LIBNAME} STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/test/snmalloc_testlib.cc) + target_link_libraries(${LIBNAME} PRIVATE snmalloc) + set_target_properties(${LIBNAME} PROPERTIES INTERFACE_LINK_LIBRARIES "") + target_compile_definitions(${LIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}") + if (NOT DEFINES STREQUAL " ") + target_compile_definitions(${LIBNAME} PRIVATE ${DEFINES}) + endif() + add_warning_flags(${LIBNAME}) + endfunction() + if(NOT (DEFINED SNMALLOC_LINKER_FLAVOUR) OR ("${SNMALLOC_LINKER_FLAVOUR}" MATCHES "^$")) # Linker not specified externally; probe to see if we can make lld work set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld -Wl,--icf=all) @@ -622,6 +639,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY) set(DEFINES " ") endif() + build_test_library(${FLAVOUR} ${DEFINES}) make_tests(${FLAVOUR} ${DEFINES}) endforeach() endif() diff --git a/src/snmalloc/ds/ds.h b/src/snmalloc/ds/ds.h index e24b1f25b..9c9cd0a49 100644 --- a/src/snmalloc/ds/ds.h +++ b/src/snmalloc/ds/ds.h @@ -6,8 +6,10 @@ #include "../ds_aal/ds_aal.h" #include "../pal/pal.h" #include "aba.h" -#include "allocconfig.h" #include "combininglock.h" #include "entropy.h" #include "mpmcstack.h" #include "pagemap.h" +#include "pool.h" +#include "pooled.h" +#include "sizeclasstable.h" diff --git a/src/snmalloc/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h index f1e965e53..c5d8c21ab 100644 --- a/src/snmalloc/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -1,8 +1,8 @@ #pragma once +#include "../ds_core/allocconfig.h" #include "../ds_core/ds_core.h" #include "aba.h" -#include "allocconfig.h" namespace snmalloc { diff --git a/src/snmalloc/mem/pool.h b/src/snmalloc/ds/pool.h similarity index 99% rename from src/snmalloc/mem/pool.h rename to src/snmalloc/ds/pool.h index 2a94ec440..ca5831b6e 100644 --- a/src/snmalloc/mem/pool.h +++ b/src/snmalloc/ds/pool.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ds.h" +#include "ds.h" #include "pooled.h" namespace snmalloc diff --git a/src/snmalloc/mem/pooled.h b/src/snmalloc/ds/pooled.h similarity index 93% rename from src/snmalloc/mem/pooled.h rename to src/snmalloc/ds/pooled.h index ca4e94101..140bbfd74 100644 --- a/src/snmalloc/mem/pooled.h +++ b/src/snmalloc/ds/pooled.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds/ds.h" -#include "backend_concept.h" +#include "ds.h" namespace snmalloc { diff --git a/src/snmalloc/mem/sizeclasstable.h b/src/snmalloc/ds/sizeclasstable.h similarity index 92% rename from src/snmalloc/mem/sizeclasstable.h rename to src/snmalloc/ds/sizeclasstable.h index d31a1f692..5db3cb5fa 100644 --- a/src/snmalloc/mem/sizeclasstable.h +++ b/src/snmalloc/ds/sizeclasstable.h @@ -1,6 +1,6 @@ #pragma once -#include "../ds/ds.h" +#include "../pal/pal.h" /** * This file contains all the code for transforming transforming sizes to @@ -15,25 +15,8 @@ namespace snmalloc { - using smallsizeclass_t = size_t; using chunksizeclass_t = size_t; - static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) - { - // Don't use sizeclasses that are not a multiple of the alignment. - // For example, 24 byte allocations can be - // problematic for some data due to alignment issues. - auto sc = static_cast( - bits::to_exp_mant_const(size)); - - SNMALLOC_ASSERT(sc == static_cast(sc)); - - return sc; - } - - constexpr size_t NUM_SMALL_SIZECLASSES = - size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1; - // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE). constexpr size_t NUM_LARGE_CLASSES = DefaultPal::address_bits - MAX_SMALL_SIZECLASS_BITS; @@ -97,7 +80,7 @@ namespace snmalloc constexpr smallsizeclass_t as_small() { SNMALLOC_ASSERT(is_small()); - return value & (TAG - 1); + return smallsizeclass_t(value & (TAG - 1)); } constexpr chunksizeclass_t as_large() @@ -198,8 +181,7 @@ namespace snmalloc { size_t max_capacity = 0; - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { auto& meta = fast_small(sizeclass); @@ -230,8 +212,7 @@ namespace snmalloc // Get maximum precision to calculate largest division range. DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity); - for (sizeclass_compress_t sizeclass = 0; - sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { // Calculate reciprocal division constant. @@ -434,7 +415,8 @@ namespace snmalloc sizeclass_compress_t sizeclass = 0; for (; sizeclass < minimum_class; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { table[sizeclass_lookup_index(curr)] = minimum_class; @@ -443,7 +425,8 @@ namespace snmalloc for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { - for (; curr <= sizeclass_metadata.fast_small(sizeclass).size; + for (; curr <= + sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size; curr += MIN_ALLOC_STEP_SIZE) { auto i = sizeclass_lookup_index(curr); @@ -457,30 +440,19 @@ namespace snmalloc constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup(); - /** - * @brief Returns true if the size is a small sizeclass. Note that - * 0 is not considered a small sizeclass. - */ - constexpr bool is_small_sizeclass(size_t size) - { - // Perform the - 1 on size, so that zero wraps around and ends up on - // slow path. - return (size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1); - } - constexpr smallsizeclass_t size_to_sizeclass(size_t size) { if (SNMALLOC_LIKELY(is_small_sizeclass(size))) { auto index = sizeclass_lookup_index(size); SNMALLOC_ASSERT(index < sizeclass_lookup_size); - return sizeclass_lookup.table[index]; + return smallsizeclass_t(sizeclass_lookup.table[index]); } // Check this is not called on large sizes. SNMALLOC_ASSERT(size == 0); // Map size == 0 to the first sizeclass. - return 0; + return smallsizeclass_t(0); } /** diff --git a/src/snmalloc/ds/allocconfig.h b/src/snmalloc/ds_core/allocconfig.h similarity index 99% rename from src/snmalloc/ds/allocconfig.h rename to src/snmalloc/ds_core/allocconfig.h index d58d863b8..5ee00af76 100644 --- a/src/snmalloc/ds/allocconfig.h +++ b/src/snmalloc/ds_core/allocconfig.h @@ -1,5 +1,8 @@ #pragma once +#include "bits.h" +#include "mitigations.h" + namespace snmalloc { // 0 intermediate bits results in power of 2 small allocs. 1 intermediate diff --git a/src/snmalloc/ds_core/defines.h b/src/snmalloc/ds_core/defines.h index 8c83210ce..45cea3d70 100644 --- a/src/snmalloc/ds_core/defines.h +++ b/src/snmalloc/ds_core/defines.h @@ -123,8 +123,11 @@ namespace snmalloc static constexpr bool Debug = true; #endif - // Forwards reference so that the platform can define how to handle errors. + // Forward references so that the platform can define how to handle + // errors and messages. Definitions are provided in pal/pal.h once + // DefaultPal is known. [[noreturn]] SNMALLOC_COLD void error(const char* const str); + void message_impl(const char* const str); } // namespace snmalloc #define TOSTRING(expr) TOSTRING2(expr) @@ -213,21 +216,21 @@ namespace snmalloc namespace snmalloc { + template + SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) + {} + /** - * Forward declaration so that this can be called before the pal header is - * included. + * Forward declaration so that this can be called before helpers.h is + * included (e.g. in SNMALLOC_ASSERT macros expanded inside bits.h). */ template [[noreturn]] inline void report_fatal_error(Args... args); /** - * Forward declaration so that this can be called before the pal header is + * Forward declaration so that this can be called before helpers.h is * included. */ template inline void message(Args... args); - - template - SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...) - {} } // namespace snmalloc diff --git a/src/snmalloc/ds_core/ds_core.h b/src/snmalloc/ds_core/ds_core.h index cc395127b..3ce0478a1 100644 --- a/src/snmalloc/ds_core/ds_core.h +++ b/src/snmalloc/ds_core/ds_core.h @@ -7,6 +7,7 @@ * snmalloc. */ +#include "allocconfig.h" #include "bits.h" #include "cheri.h" #include "concept.h" @@ -15,4 +16,5 @@ #include "mitigations.h" #include "ptrwrap.h" #include "redblacktree.h" +#include "sizeclassstatic.h" #include "tid.h" \ No newline at end of file diff --git a/src/snmalloc/ds_core/helpers.h b/src/snmalloc/ds_core/helpers.h index e5a0ca00e..7bddbfa94 100644 --- a/src/snmalloc/ds_core/helpers.h +++ b/src/snmalloc/ds_core/helpers.h @@ -2,6 +2,7 @@ #include "bits.h" #include "snmalloc/ds_core/defines.h" +#include "snmalloc/ds_core/tid.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/type_traits.h" #include "snmalloc/stl/utility.h" @@ -339,6 +340,38 @@ namespace snmalloc } }; + /** + * Report a fatal error via a PAL-specific error reporting mechanism. This + * takes a format string and a set of arguments. The format string indicates + * the remaining arguments with "{}". This could be extended later to + * support indexing fairly easily, if we ever want to localise these error + * messages. + * + * The following are supported as arguments: + * + * - Characters (`char`), printed verbatim. + * - Strings Literals (`const char*` or `const char[]`), printed verbatim. + * - Raw pointers (void*), printed as hex strings. + * - Integers (convertible to `size_t`), printed as hex strings. + * + * These types should be sufficient for allocator-related error messages. + */ + template + [[noreturn]] inline void report_fatal_error(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + error(msg.get_message()); + } + + template + inline void message(Args... args) + { + MessageBuilder msg{stl::forward(args)...}; + MessageBuilder msg_tid{ + "{}: {}", debug_get_tid(), msg.get_message()}; + message_impl(msg_tid.get_message()); + } + /** * Convenience type that has no fields / methods. */ diff --git a/src/snmalloc/ds_core/redblacktree.h b/src/snmalloc/ds_core/redblacktree.h index ec212fa23..0b616f5b2 100644 --- a/src/snmalloc/ds_core/redblacktree.h +++ b/src/snmalloc/ds_core/redblacktree.h @@ -1,5 +1,6 @@ #pragma once +#include "snmalloc/ds_core/concept.h" #include "snmalloc/stl/array.h" #include diff --git a/src/snmalloc/ds_core/sizeclassstatic.h b/src/snmalloc/ds_core/sizeclassstatic.h new file mode 100644 index 000000000..50376a1b1 --- /dev/null +++ b/src/snmalloc/ds_core/sizeclassstatic.h @@ -0,0 +1,74 @@ +#pragma once + +#include "allocconfig.h" +#include "bits.h" + +namespace snmalloc +{ + /** + * A wrapper type for small sizeclass indices. + * + * Implicitly converts TO size_t (for array indexing, comparisons, etc.) + * but does NOT implicitly convert FROM size_t — construction must be + * explicit via smallsizeclass_t(value). + */ + struct smallsizeclass_t + { + size_t raw{0}; + + constexpr smallsizeclass_t() = default; + + explicit constexpr smallsizeclass_t(size_t v) : raw(v) {} + + /// Implicit conversion to size_t. + constexpr operator size_t() const + { + return raw; + } + + /// Pre-increment. + constexpr smallsizeclass_t& operator++() + { + ++raw; + return *this; + } + + /// Post-increment. + constexpr smallsizeclass_t operator++(int) + { + auto tmp = *this; + ++raw; + return tmp; + } + }; + + static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size) + { + // Don't use sizeclasses that are not a multiple of the alignment. + // For example, 24 byte allocations can be + // problematic for some data due to alignment issues. + return smallsizeclass_t( + bits::to_exp_mant_const(size)); + } + + constexpr size_t NUM_SMALL_SIZECLASSES = + size_t(size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE)) + 1; + + static constexpr size_t sizeclass_to_size_const(smallsizeclass_t sc) + { + return bits::from_exp_mant(sc); + } + + /** + * @brief Returns true if the size is a small sizeclass. Note that + * 0 is not considered a small sizeclass. + */ + constexpr bool is_small_sizeclass(size_t size) + { + // Perform the - 1 on size, so that zero wraps around and ends up on + // slow path. + return (size - 1) < + sizeclass_to_size_const(smallsizeclass_t(NUM_SMALL_SIZECLASSES - 1)); + } + +} // namespace snmalloc diff --git a/src/snmalloc/global/globalalloc.h b/src/snmalloc/global/globalalloc.h index 110051e2f..5a80a26f3 100644 --- a/src/snmalloc/global/globalalloc.h +++ b/src/snmalloc/global/globalalloc.h @@ -17,19 +17,19 @@ namespace snmalloc // Handling the message queue for each stack is non-atomic. auto* first = AllocPool::extract(); auto* alloc = first; - decltype(alloc) last; - if (alloc != nullptr) - { - while (alloc != nullptr) - { - alloc->flush(); - last = alloc; - alloc = AllocPool::extract(alloc); - } + if (alloc == nullptr) + return; - AllocPool::restore(first, last); + decltype(alloc) last = alloc; + while (alloc != nullptr) + { + alloc->flush(); + last = alloc; + alloc = AllocPool::extract(alloc); } + + AllocPool::restore(first, last); } /** @@ -324,8 +324,18 @@ namespace snmalloc template SNMALLOC_FAST_PATH_INLINE void* alloc() { - return ThreadAlloc::get().alloc( - aligned_size(align, size)); + constexpr size_t sz = aligned_size(align, size); + if constexpr (is_small_sizeclass(sz)) + { + constexpr auto sc = size_to_sizeclass_const(sz); + return ThreadAlloc::get().template alloc( + sc); + } + else + { + return ThreadAlloc::get().template alloc( + sz); + } } template @@ -335,6 +345,17 @@ namespace snmalloc aligned_size(align, size)); } + /** + * Allocate a block for a known small sizeclass. + * The sizeclass can be computed at compile time with size_to_sizeclass_const. + */ + template + SNMALLOC_FAST_PATH_INLINE void* alloc(smallsizeclass_t sizeclass) + { + return ThreadAlloc::get().template alloc( + sizeclass); + } + template SNMALLOC_FAST_PATH_INLINE void* alloc_aligned(size_t align, size_t size) { @@ -342,12 +363,13 @@ namespace snmalloc aligned_size(align, size)); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void dealloc(void* p) { ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + dealloc(void* p, size_t size) { check_size(p, size); ThreadAlloc::get().dealloc(p); @@ -360,14 +382,15 @@ namespace snmalloc ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void dealloc(void* p, size_t size, size_t align) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + dealloc(void* p, size_t size, size_t align) { auto rsize = aligned_size(align, size); check_size(p, rsize); ThreadAlloc::get().dealloc(p); } - SNMALLOC_FAST_PATH_INLINE void debug_teardown() + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void debug_teardown() { return ThreadAlloc::teardown(); } diff --git a/src/snmalloc/global/libc.h b/src/snmalloc/global/libc.h index 99df44617..b816dfe30 100644 --- a/src/snmalloc/global/libc.h +++ b/src/snmalloc/global/libc.h @@ -19,27 +19,61 @@ namespace snmalloc::libc return err; } - inline void* __malloc_end_pointer(void* ptr) + SNMALLOC_USED_FUNCTION inline void* __malloc_end_pointer(void* ptr) { return snmalloc::external_pointer(ptr); } - SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + __malloc_start_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + __malloc_last_byte_pointer(void* ptr) + { + return snmalloc::external_pointer(ptr); + } + + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* malloc(size_t size) { return snmalloc::alloc(size); } - SNMALLOC_FAST_PATH_INLINE void free(void* ptr) + /** + * Allocate for a pre-computed small sizeclass. + * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. + */ + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + malloc_small(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + + /** + * Allocate for a pre-computed small sizeclass. + * Use is_small_sizeclass() + size_to_sizeclass_const() to get the class. + */ + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + malloc_small_zero(smallsizeclass_t sizeclass) + { + return snmalloc::alloc(sizeclass); + } + + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void free(void* ptr) { dealloc(ptr); } - SNMALLOC_FAST_PATH_INLINE void free_sized(void* ptr, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void + free_sized(void* ptr, size_t size) { dealloc(ptr, size); } - SNMALLOC_FAST_PATH_INLINE void* calloc(size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + calloc(size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -50,7 +84,8 @@ namespace snmalloc::libc return alloc(sz); } - SNMALLOC_FAST_PATH_INLINE void* realloc(void* ptr, size_t size) + SNMALLOC_USED_FUNCTION SNMALLOC_FAST_PATH_INLINE void* + realloc(void* ptr, size_t size) { // Glibc treats // realloc(p, 0) as free(p) @@ -99,12 +134,13 @@ namespace snmalloc::libc return p; } - inline size_t malloc_usable_size(const void* ptr) + SNMALLOC_USED_FUNCTION inline size_t malloc_usable_size(const void* ptr) { return alloc_size(ptr); } - inline void* reallocarray(void* ptr, size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION inline void* + reallocarray(void* ptr, size_t nmemb, size_t size) { bool overflow = false; size_t sz = bits::umul(size, nmemb, overflow); @@ -115,7 +151,8 @@ namespace snmalloc::libc return realloc(ptr, sz); } - inline int reallocarr(void* ptr_, size_t nmemb, size_t size) + SNMALLOC_USED_FUNCTION inline int + reallocarr(void* ptr_, size_t nmemb, size_t size) { int err = errno; bool overflow = false; @@ -150,7 +187,7 @@ namespace snmalloc::libc return 0; } - inline void* memalign(size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline void* memalign(size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY(alignment == 0 || !bits::is_pow2(alignment))) { @@ -160,12 +197,14 @@ namespace snmalloc::libc return alloc_aligned(alignment, size); } - inline void* aligned_alloc(size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline void* + aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } - inline int posix_memalign(void** memptr, size_t alignment, size_t size) + SNMALLOC_USED_FUNCTION inline int + posix_memalign(void** memptr, size_t alignment, size_t size) { if (SNMALLOC_UNLIKELY( (alignment < sizeof(uintptr_t) || !bits::is_pow2(alignment)))) diff --git a/src/snmalloc/mem/backend_concept.h b/src/snmalloc/mem/backend_concept.h index 9e0a21e77..524b40b9a 100644 --- a/src/snmalloc/mem/backend_concept.h +++ b/src/snmalloc/mem/backend_concept.h @@ -2,7 +2,6 @@ #ifdef __cpp_concepts # include "../ds/ds.h" -# include "sizeclasstable.h" # include diff --git a/src/snmalloc/mem/corealloc.h b/src/snmalloc/mem/corealloc.h index 5ec7bf1f3..3f4514477 100644 --- a/src/snmalloc/mem/corealloc.h +++ b/src/snmalloc/mem/corealloc.h @@ -4,9 +4,7 @@ #include "check_init.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" #include "ticker.h" @@ -607,8 +605,7 @@ namespace snmalloc { // Perform the - 1 on size, so that zero wraps around and ends up on // slow path. - if (SNMALLOC_LIKELY( - (size - 1) <= (sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1) - 1))) + if (SNMALLOC_LIKELY(is_small_sizeclass(size))) { // Small allocations are more likely. Improve // branch prediction by placing this case first. @@ -619,11 +616,29 @@ namespace snmalloc } /** - * Fast allocation for small objects. + * Allocates a block of memory for a known small sizeclass. + * @param sizeclass The sizeclass index (must be a valid small sizeclass). + * @return A pointer to the allocated block, or nullptr on failure. + * + * Callers can pre-compute the sizeclass (e.g. at compile time with + * size_to_sizeclass_const) and avoid the dynamic is_small_sizeclass + * check and sizeclass lookup. + */ + template + SNMALLOC_FAST_PATH ALLOCATOR void* + alloc(smallsizeclass_t sizeclass) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc( + sizeclass, sizeclass_to_size(sizeclass)); + } + + /** + * Fast allocation for small objects, with pre-computed sizeclass. */ template - SNMALLOC_FAST_PATH void* - small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + SNMALLOC_FAST_PATH void* small_alloc( + smallsizeclass_t sizeclass, + size_t size) noexcept(noexcept(Conts::failure(0))) { auto domesticate = [this](freelist::QueuePtr p) SNMALLOC_FAST_PATH_LAMBDA { @@ -631,7 +646,6 @@ namespace snmalloc }; auto& key = freelist::Object::key_root; - smallsizeclass_t sizeclass = size_to_sizeclass(size); auto* fl = &small_fast_free_lists[sizeclass]; if (SNMALLOC_LIKELY(!fl->empty())) { @@ -653,6 +667,17 @@ namespace snmalloc size); } + /** + * Fast allocation for small objects from a byte size. + * Computes the sizeclass and delegates to the two-arg version. + */ + template + SNMALLOC_FAST_PATH void* + small_alloc(size_t size) noexcept(noexcept(Conts::failure(0))) + { + return small_alloc(size_to_sizeclass(size), size); + } + /** * Allocation that are larger that will result in an allocation directly * from the backend. This additionally checks for 0 size allocations and @@ -1400,7 +1425,7 @@ namespace snmalloc auto& key = freelist::Object::key_root; - for (size_t i = 0; i < NUM_SMALL_SIZECLASSES; i++) + for (smallsizeclass_t i; i < NUM_SMALL_SIZECLASSES; i++) { if (small_fast_free_lists[i].empty()) continue; @@ -1421,7 +1446,7 @@ namespace snmalloc local_state, get_trunc_id()); // We may now have unused slabs, return to the global allocator. - for (smallsizeclass_t sizeclass = 0; sizeclass < NUM_SMALL_SIZECLASSES; + for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++) { dealloc_local_slabs(sizeclass); diff --git a/src/snmalloc/mem/mem.h b/src/snmalloc/mem/mem.h index 3e9212635..0bcd19aff 100644 --- a/src/snmalloc/mem/mem.h +++ b/src/snmalloc/mem/mem.h @@ -7,9 +7,6 @@ #include "entropy.h" #include "freelist.h" #include "metadata.h" -#include "pool.h" -#include "pooled.h" #include "remoteallocator.h" #include "remotecache.h" -#include "sizeclasstable.h" #include "ticker.h" diff --git a/src/snmalloc/mem/metadata.h b/src/snmalloc/mem/metadata.h index d034f5f3b..546947afa 100644 --- a/src/snmalloc/mem/metadata.h +++ b/src/snmalloc/mem/metadata.h @@ -2,7 +2,6 @@ #include "../ds/ds.h" #include "freelist.h" -#include "sizeclasstable.h" #include "snmalloc/stl/new.h" namespace snmalloc diff --git a/src/snmalloc/mem/remotecache.h b/src/snmalloc/mem/remotecache.h index 045ab141b..c92c99a65 100644 --- a/src/snmalloc/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -5,7 +5,6 @@ #include "freelist.h" #include "metadata.h" #include "remoteallocator.h" -#include "sizeclasstable.h" #include "snmalloc/stl/array.h" #include "snmalloc/stl/atomic.h" diff --git a/src/snmalloc/mem/secondary/gwp_asan.h b/src/snmalloc/mem/secondary/gwp_asan.h index 178a3eeee..716c57935 100644 --- a/src/snmalloc/mem/secondary/gwp_asan.h +++ b/src/snmalloc/mem/secondary/gwp_asan.h @@ -1,8 +1,8 @@ #pragma once #include "gwp_asan/guarded_pool_allocator.h" +#include "snmalloc/ds/sizeclasstable.h" #include "snmalloc/ds_core/defines.h" -#include "snmalloc/mem/sizeclasstable.h" #if defined(SNMALLOC_BACKTRACE_HEADER) # include SNMALLOC_BACKTRACE_HEADER #endif diff --git a/src/snmalloc/pal/pal.h b/src/snmalloc/pal/pal.h index 6fbb8f347..3b3242ac5 100644 --- a/src/snmalloc/pal/pal.h +++ b/src/snmalloc/pal/pal.h @@ -66,11 +66,17 @@ namespace snmalloc # error Unsupported platform #endif - [[noreturn]] SNMALLOC_SLOW_PATH inline void error(const char* const str) + [[noreturn]] SNMALLOC_SLOW_PATH SNMALLOC_USED_FUNCTION inline void + error(const char* const str) { DefaultPal::error(str); } + SNMALLOC_USED_FUNCTION inline void message_impl(const char* const str) + { + DefaultPal::message(str); + } + // Used to keep Superslab metadata committed. static constexpr size_t OS_PAGE_SIZE = DefaultPal::page_size; @@ -149,35 +155,4 @@ namespace snmalloc "Page size from system header does not match snmalloc config page size."); #endif - /** - * Report a fatal error via a PAL-specific error reporting mechanism. This - * takes a format string and a set of arguments. The format string indicates - * the remaining arguments with "{}". This could be extended later to - * support indexing fairly easily, if we ever want to localise these error - * messages. - * - * The following are supported as arguments: - * - * - Characters (`char`), printed verbatim. - * - Strings Literals (`const char*` or `const char[]`), printed verbatim. - * - Raw pointers (void*), printed as hex strings. - * - Integers (convertible to `size_t`), printed as hex strings. - * - * These types should be sufficient for allocator-related error messages. - */ - template - [[noreturn]] inline void report_fatal_error(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - DefaultPal::error(msg.get_message()); - } - - template - inline void message(Args... args) - { - MessageBuilder msg{stl::forward(args)...}; - MessageBuilder msg_tid{ - "{}: {}", debug_get_tid(), msg.get_message()}; - DefaultPal::message(msg_tid.get_message()); - } } // namespace snmalloc diff --git a/src/snmalloc/pal/pal_consts.h b/src/snmalloc/pal/pal_consts.h index 293ad3318..15b8dc18d 100644 --- a/src/snmalloc/pal/pal_consts.h +++ b/src/snmalloc/pal/pal_consts.h @@ -1,7 +1,6 @@ #pragma once -#include "../ds_core/ds_core.h" -#include "snmalloc/stl/atomic.h" +#include namespace snmalloc { @@ -69,7 +68,7 @@ namespace snmalloc /** * Flag indicating whether requested memory should be zeroed. */ - enum ZeroMem + enum ZeroMem : int { /** * Memory should not be zeroed, contents are undefined. diff --git a/src/snmalloc/pal/pal_windows.h b/src/snmalloc/pal/pal_windows.h index a44079dea..602749aad 100644 --- a/src/snmalloc/pal/pal_windows.h +++ b/src/snmalloc/pal/pal_windows.h @@ -592,7 +592,7 @@ namespace snmalloc # ifdef PLATFORM_HAS_VIRTUALALLOC2 template - void* PALWindows::reserve_aligned(size_t size) noexcept + inline void* PALWindows::reserve_aligned(size_t size) noexcept { SNMALLOC_ASSERT(bits::is_pow2(size)); SNMALLOC_ASSERT(size >= minimum_alloc_size); @@ -622,7 +622,7 @@ namespace snmalloc } # endif - void* PALWindows::reserve(size_t size) noexcept + inline void* PALWindows::reserve(size_t size) noexcept { void* ret = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); diff --git a/src/test/func/bits/bits.cc b/src/test/func/bits/bits.cc index 0046516a3..d1ffcf90e 100644 --- a/src/test/func/bits/bits.cc +++ b/src/test/func/bits/bits.cc @@ -3,8 +3,8 @@ */ #include -#include #include +#include void test_ctz() { diff --git a/src/test/func/first_operation/first_operation.cc b/src/test/func/first_operation/first_operation.cc index c28d381af..d375cbef0 100644 --- a/src/test/func/first_operation/first_operation.cc +++ b/src/test/func/first_operation/first_operation.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void alloc1(size_t size) @@ -47,14 +47,14 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } void calloc2(size_t size) { - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -109,7 +109,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/func/malloc/malloc.cc b/src/test/func/malloc/malloc.cc index b3ab963cd..3dcfb4559 100644 --- a/src/test/func/malloc/malloc.cc +++ b/src/test/func/malloc/malloc.cc @@ -1,12 +1,8 @@ +#include #include #include #include - -#define SNMALLOC_NAME_MANGLE(a) our_##a -#undef SNMALLOC_NO_REALLOCARRAY -#undef SNMALLOC_NO_REALLOCARR -#define SNMALLOC_BOOTSTRAP_ALLOCATOR -#include +#include using namespace snmalloc; @@ -27,8 +23,8 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) INFO("Unexpected null returned.\n"); failed = true; } - const auto alloc_size = our_malloc_usable_size(p); - auto expected_size = our_malloc_good_size(size); + const auto alloc_size = testlib_malloc_usable_size(p); + auto expected_size = testlib_malloc_good_size(size); const auto exact_size = align == 1; #ifdef __CHERI_PURE_CAPABILITY__ const auto cheri_size = __builtin_cheri_length_get(p); @@ -93,14 +89,14 @@ void check_result(size_t size, size_t align, void* p, int err, bool null) } EXPECT(!failed, "check_result failed! {}", p); - our_free(p); + testlib_free(p); } void test_calloc(size_t nmemb, size_t size, int err, bool null) { START_TEST("calloc({}, {}) combined size {}\n", nmemb, size, nmemb * size); errno = SUCCESS; - void* p = our_calloc(nmemb, size); + void* p = testlib_calloc(nmemb, size); if (p != nullptr) { @@ -116,23 +112,23 @@ void test_realloc(void* p, size_t size, int err, bool null) { size_t old_size = 0; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("realloc({}({}), {})", p, old_size, size); errno = SUCCESS; - auto new_p = our_realloc(p, size); + auto new_p = testlib_realloc(p, size); check_result(size, 1, new_p, err, null); // Realloc failure case, deallocate original block as not // handled by check_result. if (new_p == nullptr && size != 0) - our_free(p); + testlib_free(p); } void test_posix_memalign(size_t size, size_t align, int err, bool null) { START_TEST("posix_memalign(&p, {}, {})", align, size); void* p = nullptr; - errno = our_posix_memalign(&p, align, size); + errno = testlib_posix_memalign(&p, align, size); check_result(size, align, p, err, null); } @@ -140,7 +136,7 @@ void test_memalign(size_t size, size_t align, int err, bool null) { START_TEST("memalign({}, {})", align, size); errno = SUCCESS; - void* p = our_memalign(align, size); + void* p = testlib_memalign(align, size); check_result(size, align, p, err, null); } @@ -149,13 +145,13 @@ void test_reallocarray(void* p, size_t nmemb, size_t size, int err, bool null) size_t old_size = 0; size_t tsize = nmemb * size; if (p != nullptr) - old_size = our_malloc_usable_size(p); + old_size = testlib_malloc_usable_size(p); START_TEST("reallocarray({}({}), {})", p, old_size, tsize); errno = SUCCESS; - auto new_p = our_reallocarray(p, nmemb, size); + auto new_p = testlib_reallocarray(p, nmemb, size); if (new_p == nullptr && tsize != 0) - our_free(p); + testlib_free(p); check_result(tsize, 1, new_p, err, null); } @@ -165,29 +161,29 @@ void test_reallocarr( void* p = nullptr; if (size_old != (size_t)~0) - p = our_malloc(size_old); + p = testlib_malloc(size_old); START_TEST("reallocarr({}({}), {})", p, nmemb, size); errno = SUCCESS; - int r = our_reallocarr(&p, nmemb, size); + int r = testlib_reallocarr(&p, nmemb, size); EXPECT(r == err, "reallocarr failed! expected {} got {}\n", err, r); check_result(nmemb * size, 1, p, err, null); - p = our_malloc(size); + p = testlib_malloc(size); if (!p) { return; } for (size_t i = 1; i < size; i++) static_cast(p)[i] = 1; - our_reallocarr(&p, nmemb, size); + testlib_reallocarr(&p, nmemb, size); if (r != SUCCESS) - our_free(p); + testlib_free(p); for (size_t i = 1; i < size; i++) { EXPECT(static_cast(p)[i] == 1, "data consistency failed! at {}", i); } - our_free(p); + testlib_free(p); } int main(int argc, char** argv) @@ -223,31 +219,31 @@ int main(int argc, char** argv) abort(); } - our_free(nullptr); + testlib_free(nullptr); /* A very large allocation size that we expect to fail. */ const size_t too_big_size = ((size_t)-1) / 2; - check_result(too_big_size, 1, our_malloc(too_big_size), ENOMEM, true); + check_result(too_big_size, 1, testlib_malloc(too_big_size), ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); START_TEST("malloc: {}", size); errno = SUCCESS; - check_result(size, 1, our_malloc(size), SUCCESS, false); + check_result(size, 1, testlib_malloc(size), SUCCESS, false); errno = SUCCESS; - check_result(size + 1, 1, our_malloc(size + 1), SUCCESS, false); + check_result(size + 1, 1, testlib_malloc(size + 1), SUCCESS, false); } test_calloc(0, 0, SUCCESS, false); - our_free(nullptr); + testlib_free(nullptr); test_calloc(1, too_big_size, ENOMEM, true); errno = SUCCESS; - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); @@ -268,38 +264,38 @@ int main(int argc, char** argv) // Check realloc(nullptr,0) behaves like malloc(1) test_realloc(nullptr, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES; sc++) { const size_t size = sizeclass_to_size(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < NUM_SMALL_SIZECLASSES; sc2++) + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < NUM_SMALL_SIZECLASSES; sc2++) { const size_t size2 = sizeclass_to_size(sc2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } // Check realloc(p,0), behaves like free(p), if p != nullptr - test_realloc(our_malloc(size), 0, SUCCESS, true); + test_realloc(testlib_malloc(size), 0, SUCCESS, true); } - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_realloc(our_malloc(size), size, SUCCESS, false); + test_realloc(testlib_malloc(size), size, SUCCESS, false); test_realloc(nullptr, size, SUCCESS, false); - test_realloc(our_malloc(size), too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + test_realloc(testlib_malloc(size), too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); INFO("size1: {}, size2:{}\n", size, size2); - test_realloc(our_malloc(size), size2, SUCCESS, false); - test_realloc(our_malloc(size + 1), size2, SUCCESS, false); + test_realloc(testlib_malloc(size), size2, SUCCESS, false); + test_realloc(testlib_malloc(size + 1), size2, SUCCESS, false); } } - test_realloc(our_malloc(64), 4194304, SUCCESS, false); + test_realloc(testlib_malloc(64), 4194304, SUCCESS, false); test_posix_memalign(0, 0, EINVAL, true); test_posix_memalign(too_big_size, 0, EINVAL, true); @@ -311,7 +307,7 @@ int main(int argc, char** argv) // Check overflow with alignment taking it round to 0. test_memalign(1 - align, align, ENOMEM, true); - for (smallsizeclass_t sc = 0; sc < NUM_SMALL_SIZECLASSES - 6; sc++) + for (smallsizeclass_t sc(0); sc < NUM_SMALL_SIZECLASSES - 6; sc++) { const size_t size = sizeclass_to_size(sc); test_posix_memalign(size, align, SUCCESS, false); @@ -324,37 +320,37 @@ int main(int argc, char** argv) } test_reallocarray(nullptr, 1, 0, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); - test_reallocarray(our_malloc(size), 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, 0, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, 0, SUCCESS, false); test_reallocarray(nullptr, 1, size, SUCCESS, false); - test_reallocarray(our_malloc(size), 1, too_big_size, ENOMEM, true); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + test_reallocarray(testlib_malloc(size), 1, too_big_size, ENOMEM, true); + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); - test_reallocarray(our_malloc(size), 1, size2, SUCCESS, false); - test_reallocarray(our_malloc(size + 1), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size), 1, size2, SUCCESS, false); + test_reallocarray(testlib_malloc(size + 1), 1, size2, SUCCESS, false); } } test_reallocarr((size_t)~0, 1, 0, SUCCESS, false); test_reallocarr((size_t)~0, 1, 16, SUCCESS, false); - for (smallsizeclass_t sc = 0; sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) + for (smallsizeclass_t sc(0); sc < (MAX_SMALL_SIZECLASS_BITS + 4); sc++) { const size_t size = bits::one_at_bit(sc); test_reallocarr(size, 1, size, SUCCESS, false); test_reallocarr(size, 1, 0, SUCCESS, false); test_reallocarr(size, 2, size, SUCCESS, false); - void* p = our_malloc(size); + void* p = testlib_malloc(size); EXPECT(p != nullptr, "realloc alloc failed with {}", size); - int r = our_reallocarr(&p, 1, too_big_size); + int r = testlib_reallocarr(&p, 1, too_big_size); EXPECT(r == ENOMEM, "expected failure on allocation\n"); - our_free(p); + testlib_free(p); - for (smallsizeclass_t sc2 = 0; sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) + for (smallsizeclass_t sc2(0); sc2 < (MAX_SMALL_SIZECLASS_BITS + 4); sc2++) { const size_t size2 = bits::one_at_bit(sc2); START_TEST("size1: {}, size2:{}", size, size2); @@ -363,7 +359,7 @@ int main(int argc, char** argv) } EXPECT( - our_malloc_usable_size(nullptr) == 0, + testlib_malloc_usable_size(nullptr) == 0, "malloc_usable_size(nullptr) should be zero"); snmalloc::debug_check_empty(); diff --git a/src/test/func/memory/memory.cc b/src/test/func/memory/memory.cc index 7c181a24a..8d2f7c77d 100644 --- a/src/test/func/memory/memory.cc +++ b/src/test/func/memory/memory.cc @@ -1,9 +1,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -178,7 +178,7 @@ void test_calloc() memset(p, 0xFF, size); snmalloc::dealloc(p, size); - p = snmalloc::alloc(size); + p = snmalloc::alloc(size); for (size_t i = 0; i < size; i++) { @@ -239,11 +239,11 @@ void test_double_alloc() void test_external_pointer() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - size_t size = sizeclass_to_size(sc); + size_t size = sizeclass_to_size_const(sc); void* p1 = snmalloc::alloc(size); if (size != snmalloc::alloc_size(p1)) @@ -259,8 +259,8 @@ void test_external_pointer() for (size_t offset = 0; offset < size; offset += 17) { void* p2 = pointer_offset(p1, offset); - void* p3 = snmalloc::external_pointer(p2); - void* p4 = snmalloc::external_pointer(p2); + void* p3 = snmalloc::libc::__malloc_start_pointer(p2); + void* p4 = snmalloc::libc::__malloc_last_byte_pointer(p2); if (p1 != p3) { if (p3 > p1 || snmalloc::is_owned(p1)) @@ -293,7 +293,7 @@ void test_external_pointer() void check_offset(void* base, void* interior) { - void* calced_base = snmalloc::external_pointer((void*)interior); + void* calced_base = snmalloc::libc::__malloc_start_pointer((void*)interior); if (calced_base != (void*)base) { if (calced_base > base || snmalloc::is_owned(base)) @@ -320,10 +320,10 @@ void test_external_pointer_large() { xoroshiro::p128r64 r; - constexpr size_t count_log = DefaultPal::address_bits > 32 ? 5 : 3; - constexpr size_t count = 1 << count_log; + const size_t count_log = pal_address_bits() > 32 ? 5 : 3; + const size_t count = size_t(1) << count_log; // Pre allocate all the objects - size_t* objects[count]; + size_t* objects[1 << 5]; // max possible count size_t total_size = 0; @@ -376,7 +376,7 @@ void test_external_pointer_dealloc_bug() for (size_t i = 0; i < count; i++) { - snmalloc::external_pointer(allocs[i]); + snmalloc::libc::__malloc_start_pointer(allocs[i]); } snmalloc::dealloc(allocs[0]); @@ -391,10 +391,11 @@ void test_external_pointer_stack() for (size_t i = 0; i < stack.size(); i++) { - if (snmalloc::external_pointer(&stack[i]) > &stack[i]) + if (snmalloc::libc::__malloc_start_pointer(&stack[i]) > &stack[i]) { std::cout << "Stack pointer: " << &stack[i] << " external pointer: " - << snmalloc::external_pointer(&stack[i]) << std::endl; + << snmalloc::libc::__malloc_start_pointer(&stack[i]) + << std::endl; abort(); } } @@ -408,7 +409,8 @@ void test_alloc_16M() const size_t size = 16'000'000; void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -417,8 +419,9 @@ void test_calloc_16M() // sizes >= 16M use large_alloc const size_t size = 16'000'000; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -430,8 +433,9 @@ void test_calloc_large_bug() // not a multiple of page size. const size_t size = (MAX_SMALL_SIZECLASS_SIZE << 3) - 7; - void* p1 = snmalloc::alloc(size); - SNMALLOC_CHECK(snmalloc::alloc_size(snmalloc::external_pointer(p1)) >= size); + void* p1 = snmalloc::alloc(size); + SNMALLOC_CHECK( + snmalloc::alloc_size(snmalloc::libc::__malloc_start_pointer(p1)) >= size); snmalloc::dealloc(p1); } @@ -485,16 +489,16 @@ void test_static_sized_allocs() void test_remaining_bytes() { - for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE); + for (snmalloc::smallsizeclass_t sc = size_to_sizeclass_const(MIN_ALLOC_SIZE); sc < NUM_SMALL_SIZECLASSES; sc++) { - auto size = sizeclass_to_size(sc); + auto size = sizeclass_to_size_const(sc); char* p = (char*)snmalloc::alloc(size); for (size_t offset = 0; offset < size; offset++) { auto rem = - snmalloc::remaining_bytes(address_cast(pointer_offset(p, offset))); + snmalloc::remaining_bytes((address_t)pointer_offset(p, offset)); if (rem != (size - offset)) { if (rem < (size - offset) || snmalloc::is_owned(p)) diff --git a/src/test/func/memory_usage/memory_usage.cc b/src/test/func/memory_usage/memory_usage.cc index 68fa179f7..036bcfe2f 100644 --- a/src/test/func/memory_usage/memory_usage.cc +++ b/src/test/func/memory_usage/memory_usage.cc @@ -4,12 +4,9 @@ */ #include #include +#include #include -#define SNMALLOC_NAME_MANGLE(a) our_##a -#include "../../../snmalloc/override/malloc-extensions.cc" -#include "../../../snmalloc/override/malloc.cc" - using namespace snmalloc; bool print_memory_usage() @@ -43,7 +40,7 @@ void add_n_allocs(size_t n) { while (true) { - auto p = our_malloc(1024); + auto p = testlib_malloc(1024); allocs.push_back(p); if (print_memory_usage()) { @@ -64,7 +61,7 @@ void remove_n_allocs(size_t n) if (allocs.empty()) return; auto p = allocs.back(); - our_free(p); + testlib_free(p); allocs.pop_back(); if (print_memory_usage()) { diff --git a/src/test/func/multi_atexit/multi_atexit.cc b/src/test/func/multi_atexit/multi_atexit.cc index e4f5ed327..09919ada3 100644 --- a/src/test/func/multi_atexit/multi_atexit.cc +++ b/src/test/func/multi_atexit/multi_atexit.cc @@ -13,8 +13,8 @@ #endif #ifdef RUN_TEST -# include # include +# include void do_nothing() {} diff --git a/src/test/func/multi_threadatexit/multi_threadatexit.cc b/src/test/func/multi_threadatexit/multi_threadatexit.cc index 85f918f93..458c95aae 100644 --- a/src/test/func/multi_threadatexit/multi_threadatexit.cc +++ b/src/test/func/multi_threadatexit/multi_threadatexit.cc @@ -15,8 +15,9 @@ #endif #ifdef RUN_TEST -# include +# include # include +# include template void thread_destruct() diff --git a/src/test/func/pagemap/pagemap.cc b/src/test/func/pagemap/pagemap.cc index a0b53689f..7a03fa1a7 100644 --- a/src/test/func/pagemap/pagemap.cc +++ b/src/test/func/pagemap/pagemap.cc @@ -7,8 +7,9 @@ */ #include -#include +#include #include +#include using namespace snmalloc; static constexpr size_t GRANULARITY_BITS = 20; diff --git a/src/test/func/pool/pool.cc b/src/test/func/pool/pool.cc index 6a2874215..8f11ff689 100644 --- a/src/test/func/pool/pool.cc +++ b/src/test/func/pool/pool.cc @@ -1,8 +1,8 @@ #include #include -#include -#include +#include #include +#include #include using namespace snmalloc; @@ -56,7 +56,7 @@ void test_alloc() auto ptr = PoolA::acquire(); SNMALLOC_CHECK(ptr != nullptr); // Pool allocations should not be visible to debug_check_empty. - snmalloc::debug_check_empty(); + snmalloc::debug_check_empty(); PoolA::release(ptr); } @@ -242,13 +242,7 @@ void test_sort() int main(int argc, char** argv) { setup(); -#ifdef USE_SYSTEMATIC_TESTING - opt::Opt opt(argc, argv); - size_t seed = opt.is("--seed", 0); - Virtual::systematic_bump_ptr() += seed << 17; -#else UNUSED(argc, argv); -#endif test_double_alloc(); std::cout << "test_double_alloc passed" << std::endl; diff --git a/src/test/func/protect_fork/protect_fork.cc b/src/test/func/protect_fork/protect_fork.cc index 77227b202..b7ca4b3ff 100644 --- a/src/test/func/protect_fork/protect_fork.cc +++ b/src/test/func/protect_fork/protect_fork.cc @@ -8,9 +8,12 @@ int main() #else # define SNMALLOC_PTHREAD_FORK_PROTECTION +# include "test/snmalloc_testlib.h" + # include -# include +# include # include +# include void simulate_allocation() { diff --git a/src/test/func/redblack/redblack.cc b/src/test/func/redblack/redblack.cc index 164a5978f..61fccb6d3 100644 --- a/src/test/func/redblack/redblack.cc +++ b/src/test/func/redblack/redblack.cc @@ -11,7 +11,9 @@ # define SNMALLOC_TRACING #endif // Redblack tree needs some libraries with trace enabled. -#include "snmalloc/snmalloc.h" +#include "test/snmalloc_testlib.h" + +#include struct NodeRef { diff --git a/src/test/func/release-rounding/rounding.cc b/src/test/func/release-rounding/rounding.cc index f9541331c..4d11eaafb 100644 --- a/src/test/func/release-rounding/rounding.cc +++ b/src/test/func/release-rounding/rounding.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include using namespace snmalloc; // Check for all sizeclass that we correctly round every offset within @@ -17,9 +18,10 @@ int main(int argc, char** argv) bool failed = false; - for (size_t size_class = 0; size_class < NUM_SMALL_SIZECLASSES; size_class++) + for (smallsizeclass_t size_class; size_class < NUM_SMALL_SIZECLASSES; + size_class++) { - size_t rsize = sizeclass_to_size((uint8_t)size_class); + size_t rsize = sizeclass_to_size(size_class); size_t max_offset = sizeclass_to_slab_size(size_class); sizeclass_t sc = sizeclass_t::from_small_class(size_class); for (size_t offset = 0; offset < max_offset; offset++) diff --git a/src/test/func/sizeclass/sizeclass.cc b/src/test/func/sizeclass/sizeclass.cc index 836c62111..ac7ec6bd8 100644 --- a/src/test/func/sizeclass/sizeclass.cc +++ b/src/test/func/sizeclass/sizeclass.cc @@ -1,6 +1,7 @@ #include -#include +#include #include +#include NOINLINE snmalloc::smallsizeclass_t size_to_sizeclass(size_t size) @@ -17,9 +18,7 @@ void test_align_size() SNMALLOC_CHECK(snmalloc::aligned_size(128, 160) == 256); - for (size_t size = 1; - size < snmalloc::sizeclass_to_size(snmalloc::NUM_SMALL_SIZECLASSES - 1); - size++) + for (size_t size = 1; snmalloc::is_small_sizeclass(size); size++) { size_t rsize = snmalloc::round_size(size); @@ -85,7 +84,7 @@ int main(int, char**) std::cout << "sizeclass |-> [size_low, size_high] " << std::endl; size_t slab_size = 0; - for (snmalloc::smallsizeclass_t sz = 0; sz < snmalloc::NUM_SMALL_SIZECLASSES; + for (snmalloc::smallsizeclass_t sz(0); sz < snmalloc::NUM_SMALL_SIZECLASSES; sz++) { if ( diff --git a/src/test/func/statistics/stats.cc b/src/test/func/statistics/stats.cc index d66f060a1..97342d8fc 100644 --- a/src/test/func/statistics/stats.cc +++ b/src/test/func/statistics/stats.cc @@ -1,5 +1,6 @@ +#include #include -#include +#include #include template @@ -54,7 +55,7 @@ void debug_check_empty_2() bool result; std::vector allocs; // 1GB of allocations - size_t count = snmalloc::bits::min(2048, 1024 * 1024 * 1024 / size); + size_t count = std::min(2048, 1024 * 1024 * 1024 / size); for (size_t i = 0; i < count; i++) { diff --git a/src/test/func/teardown/teardown.cc b/src/test/func/teardown/teardown.cc index b20d8b1cf..02129374d 100644 --- a/src/test/func/teardown/teardown.cc +++ b/src/test/func/teardown/teardown.cc @@ -8,7 +8,7 @@ #include "test/setup.h" #include -#include +#include #include void trigger_teardown() @@ -59,7 +59,7 @@ void check_calloc(void* p, size_t size) void calloc1(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r); } @@ -67,7 +67,7 @@ void calloc1(size_t size) void calloc2(size_t size) { trigger_teardown(); - void* r = snmalloc::alloc(size); + void* r = snmalloc::alloc(size); check_calloc(r, size); snmalloc::dealloc(r, size); } @@ -124,7 +124,7 @@ int main(int, char**) f(5); f(7); printf("\n"); - for (size_t exp = 1; exp < snmalloc::MAX_SMALL_SIZECLASS_BITS; exp++) + for (size_t exp = 1; exp < snmalloc::max_small_sizeclass_bits(); exp++) { auto shifted = [exp](size_t v) { return v << exp; }; diff --git a/src/test/perf/contention/contention.cc b/src/test/perf/contention/contention.cc index 6bed00545..4d7b53cab 100644 --- a/src/test/perf/contention/contention.cc +++ b/src/test/perf/contention/contention.cc @@ -6,7 +6,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/test/perf/external_pointer/externalpointer.cc b/src/test/perf/external_pointer/externalpointer.cc index 76be4c708..07e69cef9 100644 --- a/src/test/perf/external_pointer/externalpointer.cc +++ b/src/test/perf/external_pointer/externalpointer.cc @@ -1,6 +1,6 @@ -#include #include #include +#include #include #include @@ -19,7 +19,7 @@ namespace test { size_t rand = (size_t)r.next(); size_t offset = bits::clz(rand); - if constexpr (DefaultPal::address_bits > 32) + if (snmalloc::pal_address_bits() > 32) { if (offset > 30) offset = 30; @@ -80,7 +80,8 @@ namespace test size_t size = *external_ptr; size_t offset = (size >> 4) * (rand & 15); void* interior_ptr = pointer_offset(external_ptr, offset); - void* calced_external = snmalloc::external_pointer(interior_ptr); + void* calced_external = + snmalloc::libc::__malloc_start_pointer(interior_ptr); if (calced_external != external_ptr) { abort(); diff --git a/src/test/perf/large_alloc/large_alloc.cc b/src/test/perf/large_alloc/large_alloc.cc new file mode 100644 index 000000000..b0f0f2bc8 --- /dev/null +++ b/src/test/perf/large_alloc/large_alloc.cc @@ -0,0 +1,83 @@ +#include +#include +#include + +using namespace snmalloc; + +static constexpr size_t ALLOC_SIZE = 800 * 1024; // 800 KB +static constexpr size_t ITERATIONS = 100000; + +void test_alloc_dealloc_cycle() +{ + { + MeasureTime m; + m << "Alloc/dealloc 800KB x " << ITERATIONS; + + for (size_t i = 0; i < ITERATIONS; i++) + { + void* p = snmalloc::alloc(ALLOC_SIZE); + SNMALLOC_CHECK(p != nullptr); + snmalloc::dealloc(p); + } + } + + snmalloc::debug_check_empty(); +} + +void test_batch_alloc_then_dealloc() +{ + static constexpr size_t BATCH = 128; + + void* ptrs[BATCH]; + + MeasureTime m; + m << "Batch alloc then dealloc 800KB x " << BATCH; + for (size_t j = 0; j < ITERATIONS / BATCH; j++) + { + for (size_t i = 0; i < BATCH; i++) + { + ptrs[i] = snmalloc::alloc(ALLOC_SIZE); + SNMALLOC_CHECK(ptrs[i] != nullptr); + } + + for (size_t i = 0; i < BATCH; i++) + { + snmalloc::dealloc(ptrs[i]); + } + } + + snmalloc::debug_check_empty(); +} + +void test_alloc_dealloc_with_touch() +{ + { + MeasureTime m; + m << "Alloc/touch/dealloc 800KB x " << ITERATIONS; + + for (size_t i = 0; i < ITERATIONS; i++) + { + char* p = static_cast(snmalloc::alloc(ALLOC_SIZE)); + SNMALLOC_CHECK(p != nullptr); + // Touch every 4KiB and last bytes to ensure pages are faulted in + for (size_t offset = 0; offset < ALLOC_SIZE; offset += 4096) + { + p[offset] = 1; + } + snmalloc::dealloc(p); + } + } + + snmalloc::debug_check_empty(); +} + +int main(int, char**) +{ + setup(); + + test_alloc_dealloc_cycle(); + test_batch_alloc_then_dealloc(); + test_alloc_dealloc_with_touch(); + + return 0; +} diff --git a/src/test/perf/lotsofthreads/lotsofthread.cc b/src/test/perf/lotsofthreads/lotsofthread.cc index 000af6534..9705dfff3 100644 --- a/src/test/perf/lotsofthreads/lotsofthread.cc +++ b/src/test/perf/lotsofthreads/lotsofthread.cc @@ -19,7 +19,7 @@ #include using namespace std; -#include +#include #define malloc snmalloc::libc::malloc #define free snmalloc::libc::free #define malloc_usable_size snmalloc::libc::malloc_usable_size diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index c9e973cc0..17e3b6509 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,8 +1,9 @@ #include #include -#include +#include #include #include +#include #include #include diff --git a/src/test/perf/post_teardown/post-teardown.cc b/src/test/perf/post_teardown/post-teardown.cc index ca9296d71..7da8605e5 100644 --- a/src/test/perf/post_teardown/post-teardown.cc +++ b/src/test/perf/post_teardown/post-teardown.cc @@ -1,7 +1,7 @@ #include -#include #include #include +#include #include using namespace snmalloc; @@ -11,7 +11,7 @@ void fill(std::vector& out, size_t count, size_t size) out.reserve(count); for (size_t i = 0; i < count; i++) { - out.push_back(snmalloc::alloc(size)); + out.push_back(snmalloc::alloc(size)); } } @@ -41,7 +41,7 @@ int main(int, char**) // Simulate the allocator already being torn down before remaining frees // (post-main / static destruction path from #809). - ThreadAlloc::teardown(); + snmalloc::debug_teardown(); fill(ptrs, alloc_count, obj_size); drain("Immediate dealloc after teardown", ptrs, obj_size); diff --git a/src/test/perf/singlethread/singlethread.cc b/src/test/perf/singlethread/singlethread.cc index ed068a31f..bf173969d 100644 --- a/src/test/perf/singlethread/singlethread.cc +++ b/src/test/perf/singlethread/singlethread.cc @@ -1,25 +1,25 @@ -#include #include #include +#include #include using namespace snmalloc; -template +template void test_alloc_dealloc(size_t count, size_t size, bool write) { { MeasureTime m; m << "Count: " << std::setw(6) << count << ", Size: " << std::setw(6) - << size - << ", ZeroMem: " << std::is_same_v << ", Write: " << write; + << size << ", ZeroMem: " << (zero_mem == ZeroMem::YesZero) + << ", Write: " << write; std::unordered_set set; // alloc 1.5x objects for (size_t i = 0; i < ((count * 3) / 2); i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -41,7 +41,7 @@ void test_alloc_dealloc(size_t count, size_t size, bool write) // alloc 1x objects for (size_t i = 0; i < count; i++) { - void* p = snmalloc::alloc(size); + void* p = snmalloc::alloc(size); SNMALLOC_CHECK(set.find(p) == set.end()); if (write) @@ -68,18 +68,18 @@ int main(int, char**) for (size_t size = 16; size <= 128; size <<= 1) { - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); - test_alloc_dealloc(1 << 15, size, false); - test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); + test_alloc_dealloc(1 << 15, size, false); + test_alloc_dealloc(1 << 15, size, true); } for (size_t size = 1 << 12; size <= 1 << 17; size <<= 1) { - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); - test_alloc_dealloc(1 << 10, size, false); - test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); + test_alloc_dealloc(1 << 10, size, false); + test_alloc_dealloc(1 << 10, size, true); } return 0; diff --git a/src/test/perf/startup/startup.cc b/src/test/perf/startup/startup.cc index dfc6cb0c2..30fa23438 100644 --- a/src/test/perf/startup/startup.cc +++ b/src/test/perf/startup/startup.cc @@ -5,8 +5,7 @@ #include #include -#include -#include +#include #include #include @@ -31,18 +30,18 @@ class ParallelTest auto prev = ready.fetch_add(1); if (prev + 1 == cores) { - start = DefaultPal::tick(); + start = snmalloc::pal_tick(); flag = true; } while (!flag) - Aal::pause(); + snmalloc::pal_pause(); f(id); prev = complete.fetch_add(1); if (prev + 1 == cores) { - end = DefaultPal::tick(); + end = snmalloc::pal_tick(); } } @@ -77,9 +76,9 @@ int main() ParallelTest test( [](size_t id) { - auto start = DefaultPal::tick(); + auto start = snmalloc::pal_tick(); snmalloc::dealloc(snmalloc::alloc(1)); - auto end = DefaultPal::tick(); + auto end = snmalloc::pal_tick(); counters[id] = end - start; }, nthreads); diff --git a/src/test/snmalloc_testlib.cc b/src/test/snmalloc_testlib.cc new file mode 100644 index 000000000..e6b7e67b7 --- /dev/null +++ b/src/test/snmalloc_testlib.cc @@ -0,0 +1,216 @@ +/** + * @file snmalloc_testlib.cc + * @brief Single translation unit that compiles the full snmalloc allocator + * and provides symbols for test-library consumers. + * + * Built once per flavour (fast / check) into a static library that tests + * link against, avoiding redundant recompilation of the allocator in every + * test TU. + * + * Three kinds of symbols are provided: + * + * A. Non-templated inline functions (dealloc, debug_teardown, libc::*) + * are marked SNMALLOC_USED_FUNCTION in their declarations, so weak/COMDAT + * symbols are emitted in every TU that includes them; the linker resolves + * test references to the copies in this archive. + * + * B. Templates on user-visible params only (alloc): + * explicit instantiation definitions force emission of strong symbols. + * + * C. Config-templated functions: non-template overloads that call through + * to the template version. These are distinct functions (not + * ODR-conflicting with the templates). + */ + +#include + +namespace snmalloc +{ + template + void* alloc(size_t size) + { + if constexpr (zero_mem == ZeroMem::YesZero) + { + return alloc(size); + } + else + { + return alloc(size); + } + } + + template void* alloc(size_t size); + template void* alloc(size_t size); + + // -- C: Non-template wrappers for Config-templated functions ------------- + + // Config-templated functions: + + size_t alloc_size(const void* p) + { + return alloc_size(p); + } + + size_t remaining_bytes(address_t p) + { + return remaining_bytes(p); + } + + bool is_owned(void* p) + { + return is_owned(p); + } + + void debug_check_empty(bool* result) + { + debug_check_empty(result); + } + + void debug_in_use(size_t count) + { + debug_in_use(count); + } + + void cleanup_unused() + { + cleanup_unused(); + } + + // -- Opaque scoped allocator (inherits from ScopedAllocator<>) ----------- + + struct TestScopedAllocator : public ScopedAllocator<> + {}; + + TestScopedAllocator* create_scoped_allocator() + { + return new TestScopedAllocator(); + } + + void destroy_scoped_allocator(TestScopedAllocator* p) + { + delete p; + } + + void* scoped_alloc(TestScopedAllocator* a, size_t size) + { + return a->alloc->alloc(size); + } + + // ScopedAllocHandle is declared in snmalloc_testlib.h; define methods here. + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + void dealloc(void* p); + void dealloc(void* p, size_t size); + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle::ScopedAllocHandle() : ptr(create_scoped_allocator()) {} + + ScopedAllocHandle::~ScopedAllocHandle() + { + destroy_scoped_allocator(ptr); + } + + void* ScopedAllocHandle::alloc(size_t size) + { + return scoped_alloc(ptr, size); + } + + void ScopedAllocHandle::dealloc(void* p) + { + snmalloc::dealloc(p); + } + + void ScopedAllocHandle::dealloc(void* p, size_t size) + { + snmalloc::dealloc(p, size); + } + + ScopedAllocHandle* ScopedAllocHandle::operator->() + { + return this; + } + + const ScopedAllocHandle* ScopedAllocHandle::operator->() const + { + return this; + } + + ScopedAllocHandle get_scoped_allocator() + { + return {}; + } + + size_t max_small_sizeclass_bits() + { + return MAX_SMALL_SIZECLASS_BITS; + } + + // -- PAL/AAL wrappers ---------------------------------------------------- + + size_t pal_address_bits() + { + return DefaultPal::address_bits; + } + + uint64_t pal_tick() + { + return DefaultPal::tick(); + } + + void pal_pause() + { + Aal::pause(); + } + + // -- Force emission of inline functions for MSVC (where USED_FUNCTION is + // empty). Taking the address of each inline function forces the compiler to + // emit a definition in this TU, making it available to testlib consumers. + + SNMALLOC_USED_FUNCTION static const volatile void* force_emit_global[] = { + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast( + static_cast(&dealloc)), + reinterpret_cast(&debug_teardown), + }; + + namespace libc + { + SNMALLOC_USED_FUNCTION static const volatile void* force_emit_libc[] = { + reinterpret_cast(&__malloc_start_pointer), + reinterpret_cast(&__malloc_last_byte_pointer), + reinterpret_cast(&__malloc_end_pointer), + reinterpret_cast(&malloc), + reinterpret_cast(&free), + reinterpret_cast( + static_cast(&calloc)), + reinterpret_cast( + static_cast(&realloc)), + reinterpret_cast(&malloc_usable_size), + reinterpret_cast(&memalign), + reinterpret_cast(&aligned_alloc), + reinterpret_cast(&posix_memalign), + reinterpret_cast(&malloc_small), + reinterpret_cast(&malloc_small_zero), + }; + } // namespace libc +} // namespace snmalloc + +// -- override/malloc.cc symbols with testlib_ prefix ----------------------- +#undef SNMALLOC_NO_REALLOCARRAY +#undef SNMALLOC_NO_REALLOCARR +#define SNMALLOC_NAME_MANGLE(a) testlib_##a +#include + +// malloc-extensions (unprefixed - get_malloc_info_v1) +#include diff --git a/src/test/snmalloc_testlib.h b/src/test/snmalloc_testlib.h new file mode 100644 index 000000000..0ca66d255 --- /dev/null +++ b/src/test/snmalloc_testlib.h @@ -0,0 +1,188 @@ +#pragma once +/** + * @file snmalloc_testlib.h + * @brief Thin test-library header that replaces for + * tests that can be linked against a pre-compiled static library. + * + * Does NOT include snmalloc_core.h — only forward-declares the minimal types + * needed for the API surface. Tests that need snmalloc_core types (bits::*, + * sizeclass, Pal, etc.) should include themselves + * before this header. + * + * Templates on user-visible params (Conts, align) are declared and backed + * by explicit instantiations in testlib.cc. + * + * Config-templated functions are provided as non-template overloads + * (distinct from the templates, no ODR conflict) defined in testlib.cc. + */ + +#include +#include +#include +#include +#include +#include +#include + +namespace snmalloc +{ + // Forward declarations sufficient for the API surface. + // address_t: uintptr_t on all non-CHERI platforms. + using address_t = uintptr_t; + + template + class DefaultConts; + // Uninit / Zero are usable as template arguments even without the full + // definition of DefaultConts (only needed inside the allocator itself). + using Uninit = DefaultConts; + using Zero = DefaultConts; + + enum Boundary + { + Start, + End, + OnePastEnd + }; + + // -- Non-template functions (symbol forced in testlib.cc) ---------------- + void dealloc(void* p); + void dealloc(void* p, size_t size); + void dealloc(void* p, size_t size, size_t align); + + template + inline void dealloc(void* p) + { + dealloc(p, size); + } + + void debug_teardown(); + + // -- Non-template wrappers for Config-templated functions ---------------- + size_t alloc_size(const void* p); + size_t remaining_bytes(address_t p); + bool is_owned(void* p); + void debug_check_empty(bool* result = nullptr); + void debug_in_use(size_t count); + void cleanup_unused(); + + // -- Opaque scoped allocator + // ------------------------------------------------- TestScopedAllocator + // inherits from ScopedAllocator<> in testlib.cc. Forward-declared here; + // usable through ScopedAllocHandle. + struct TestScopedAllocator; + TestScopedAllocator* create_scoped_allocator(); + void destroy_scoped_allocator(TestScopedAllocator*); + void* scoped_alloc(TestScopedAllocator*, size_t size); + + struct ScopedAllocHandle + { + TestScopedAllocator* ptr; + + ScopedAllocHandle(); + ~ScopedAllocHandle(); + ScopedAllocHandle(const ScopedAllocHandle&) = delete; + ScopedAllocHandle& operator=(const ScopedAllocHandle&) = delete; + void* alloc(size_t size); + + void dealloc(void* p); + void dealloc(void* p, size_t size); + + ScopedAllocHandle* operator->(); + const ScopedAllocHandle* operator->() const; + }; + + ScopedAllocHandle get_scoped_allocator(); + + // -- Constants exposed from allocconfig.h -------------------------------- + size_t max_small_sizeclass_bits(); + + // -- PAL/AAL wrappers (avoids needing pal.h in tests) -------------------- + size_t pal_address_bits(); + uint64_t pal_tick(); + void pal_pause(); + + // Simple pointer arithmetic (matches snmalloc::pointer_offset for void*). + // Defined inline so tests that don't include aal/address.h can still use it. + inline void* pointer_offset(void* base, size_t diff) + { + return static_cast(static_cast(base) + diff); + } + + template + void* alloc(size_t size); + extern template void* alloc(size_t); + extern template void* alloc(size_t); + + template + void* alloc_aligned(size_t align, size_t size); + + // -- libc namespace ------------------------------------------------------ + namespace libc + { + void* malloc(size_t size); + void* malloc_small(smallsizeclass_t sizeclass); + void* malloc_small_zero(smallsizeclass_t sizeclass); + void free(void* ptr); + void* calloc(size_t nmemb, size_t size); + void* realloc(void* ptr, size_t size); + size_t malloc_usable_size(const void* ptr); + void* memalign(size_t alignment, size_t size); + void* aligned_alloc(size_t alignment, size_t size); + int posix_memalign(void** memptr, size_t alignment, size_t size); + void* reallocarray(void* ptr, size_t nmemb, size_t size); + int reallocarr(void* ptr_, size_t nmemb, size_t size); + void* __malloc_start_pointer(void* ptr); + void* __malloc_last_byte_pointer(void* ptr); + void* __malloc_end_pointer(void* ptr); + } // namespace libc + + /** + * Compile-time size alloc. When the size (after alignment) is a small + * sizeclass the sizeclass is computed at compile time and the allocation + * goes straight to the sizeclass-based fast path. Otherwise falls back + * to the dynamic alloc. + */ + template + inline void* alloc() + { + if constexpr (is_small_sizeclass(size)) + { + constexpr auto sc = size_to_sizeclass_const(size); + if constexpr (zero_mem == ZeroMem::YesZero) + { + return libc::malloc_small_zero(sc); + } + else + { + return libc::malloc_small(sc); + } + } + else + { + return alloc(size); + } + } +} // namespace snmalloc + +// -- malloc-extensions struct and function ---------------------------------- +#include + +// -- override/malloc.cc functions with testlib_ prefix ---------------------- +#ifndef MALLOC_USABLE_SIZE_QUALIFIER +# define MALLOC_USABLE_SIZE_QUALIFIER +#endif +extern "C" +{ + void* testlib_malloc(size_t size); + void testlib_free(void* ptr); + void testlib_cfree(void* ptr); + void* testlib_calloc(size_t nmemb, size_t size); + size_t testlib_malloc_usable_size(MALLOC_USABLE_SIZE_QUALIFIER void* ptr); + size_t testlib_malloc_good_size(size_t size); + void* testlib_realloc(void* ptr, size_t size); + void* testlib_reallocarray(void* ptr, size_t nmemb, size_t size); + int testlib_reallocarr(void* ptr, size_t nmemb, size_t size); + void* testlib_memalign(size_t alignment, size_t size); + void* testlib_aligned_alloc(size_t alignment, size_t size); + int testlib_posix_memalign(void** memptr, size_t alignment, size_t size); +}