Make test compile faster.

BUILD.bazel

@@ -6,6 +6,7 @@ filegroup(
         [
             "src/snmalloc/**/*",
             "src/test/*.h",
+            "src/test/*.cc",
             "CMakeLists.txt",
         ],
     ),

CMakeLists.txt

@@ -449,6 +449,11 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
           target_compile_definitions(${TESTNAME} PRIVATE ${DEFINES})
         endif()
 
+        # Link the pre-compiled test library into all tests.  For tests
+        # that include snmalloc.h directly the linker will simply discard
+        # the duplicate inline definitions (ODR-safe).
+        target_link_libraries(${TESTNAME} snmalloc-testlib-${TAG})
+
         if (${TEST} MATCHES "release-.*")
           message(VERBOSE "Adding test: ${TESTNAME} only for release configs")
           add_test(NAME ${TESTNAME} COMMAND ${TESTNAME} CONFIGURATIONS "Release")
@@ -480,6 +485,18 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
     endforeach()
   endfunction()
 
+  function(build_test_library TAG DEFINES)
+    set(LIBNAME snmalloc-testlib-${TAG})
+    add_library(${LIBNAME} STATIC ${CMAKE_CURRENT_SOURCE_DIR}/src/test/snmalloc_testlib.cc)
+    target_link_libraries(${LIBNAME} PRIVATE snmalloc)
+    set_target_properties(${LIBNAME} PROPERTIES INTERFACE_LINK_LIBRARIES "")
+    target_compile_definitions(${LIBNAME} PRIVATE "SNMALLOC_USE_${TEST_CLEANUP}")
+    if (NOT DEFINES STREQUAL " ")
+      target_compile_definitions(${LIBNAME} PRIVATE ${DEFINES})
+    endif()
+    add_warning_flags(${LIBNAME})
+  endfunction()
+
   if(NOT (DEFINED SNMALLOC_LINKER_FLAVOUR) OR ("${SNMALLOC_LINKER_FLAVOUR}" MATCHES "^$"))
     # Linker not specified externally; probe to see if we can make lld work
     set(CMAKE_REQUIRED_LINK_OPTIONS -fuse-ld=lld -Wl,--icf=all)
@@ -622,6 +639,7 @@ if(NOT SNMALLOC_HEADER_ONLY_LIBRARY)
         set(DEFINES " ")
       endif()
 
+      build_test_library(${FLAVOUR} ${DEFINES})
       make_tests(${FLAVOUR} ${DEFINES})
     endforeach()
   endif()

src/snmalloc/ds/ds.h

@@ -6,8 +6,10 @@
 #include "../ds_aal/ds_aal.h"
 #include "../pal/pal.h"
 #include "aba.h"
-#include "allocconfig.h"
 #include "combininglock.h"
 #include "entropy.h"
 #include "mpmcstack.h"
 #include "pagemap.h"
+#include "pool.h"
+#include "pooled.h"
+#include "sizeclasstable.h"

src/snmalloc/ds/mpmcstack.h

@@ -1,8 +1,8 @@
 #pragma once
 
+#include "../ds_core/allocconfig.h"
 #include "../ds_core/ds_core.h"
 #include "aba.h"
-#include "allocconfig.h"
 
 namespace snmalloc
 {

src/snmalloc/mem/pool.h → src/snmalloc/ds/pool.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../ds/ds.h"
+#include "ds.h"
 #include "pooled.h"
 
 namespace snmalloc

src/snmalloc/mem/pooled.h → src/snmalloc/ds/pooled.h

@@ -1,7 +1,6 @@
 #pragma once
 
-#include "../ds/ds.h"
-#include "backend_concept.h"
+#include "ds.h"
 
 namespace snmalloc
 {

src/snmalloc/mem/sizeclasstable.h → src/snmalloc/ds/sizeclasstable.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "../ds/ds.h"
+#include "../pal/pal.h"
 
 /**
  * This file contains all the code for transforming transforming sizes to
@@ -15,25 +15,8 @@
 
 namespace snmalloc
 {
-  using smallsizeclass_t = size_t;
   using chunksizeclass_t = size_t;
 
-  static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size)
-  {
-    // Don't use sizeclasses that are not a multiple of the alignment.
-    // For example, 24 byte allocations can be
-    // problematic for some data due to alignment issues.
-    auto sc = static_cast<smallsizeclass_t>(
-      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(size));
-
-    SNMALLOC_ASSERT(sc == static_cast<uint8_t>(sc));
-
-    return sc;
-  }
-
-  constexpr size_t NUM_SMALL_SIZECLASSES =
-    size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE) + 1;
-
   // Large classes range from [MAX_SMALL_SIZECLASS_SIZE, ADDRESS_SPACE).
   constexpr size_t NUM_LARGE_CLASSES =
     DefaultPal::address_bits - MAX_SMALL_SIZECLASS_BITS;
@@ -97,7 +80,7 @@ namespace snmalloc
     constexpr smallsizeclass_t as_small()
     {
       SNMALLOC_ASSERT(is_small());
-      return value & (TAG - 1);
+      return smallsizeclass_t(value & (TAG - 1));
     }
 
     constexpr chunksizeclass_t as_large()
@@ -198,8 +181,7 @@ namespace snmalloc
     {
       size_t max_capacity = 0;
 
-      for (sizeclass_compress_t sizeclass = 0;
-           sizeclass < NUM_SMALL_SIZECLASSES;
+      for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES;
            sizeclass++)
       {
         auto& meta = fast_small(sizeclass);
@@ -230,8 +212,7 @@ namespace snmalloc
       // Get maximum precision to calculate largest division range.
       DIV_MULT_SHIFT = bits::BITS - bits::next_pow2_bits_const(max_capacity);
 
-      for (sizeclass_compress_t sizeclass = 0;
-           sizeclass < NUM_SMALL_SIZECLASSES;
+      for (smallsizeclass_t sizeclass(0); sizeclass < NUM_SMALL_SIZECLASSES;
            sizeclass++)
       {
         // Calculate reciprocal division constant.
@@ -434,7 +415,8 @@ namespace snmalloc
       sizeclass_compress_t sizeclass = 0;
       for (; sizeclass < minimum_class; sizeclass++)
       {
-        for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
+        for (; curr <=
+             sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size;
              curr += MIN_ALLOC_STEP_SIZE)
         {
           table[sizeclass_lookup_index(curr)] = minimum_class;
@@ -443,7 +425,8 @@ namespace snmalloc
 
       for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++)
       {
-        for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
+        for (; curr <=
+             sizeclass_metadata.fast_small(smallsizeclass_t(sizeclass)).size;
              curr += MIN_ALLOC_STEP_SIZE)
         {
           auto i = sizeclass_lookup_index(curr);
@@ -457,30 +440,19 @@ namespace snmalloc
 
   constexpr SizeClassLookup sizeclass_lookup = SizeClassLookup();
 
-  /**
-   * @brief Returns true if the size is a small sizeclass. Note that
-   * 0 is not considered a small sizeclass.
-   */
-  constexpr bool is_small_sizeclass(size_t size)
-  {
-    // Perform the - 1 on size, so that zero wraps around and ends up on
-    // slow path.
-    return (size - 1) < sizeclass_to_size(NUM_SMALL_SIZECLASSES - 1);
-  }
-
   constexpr smallsizeclass_t size_to_sizeclass(size_t size)
   {
     if (SNMALLOC_LIKELY(is_small_sizeclass(size)))
     {
       auto index = sizeclass_lookup_index(size);
       SNMALLOC_ASSERT(index < sizeclass_lookup_size);
-      return sizeclass_lookup.table[index];
+      return smallsizeclass_t(sizeclass_lookup.table[index]);
     }
 
     // Check this is not called on large sizes.
     SNMALLOC_ASSERT(size == 0);
     // Map size == 0 to the first sizeclass.
-    return 0;
+    return smallsizeclass_t(0);
   }
 
   /**

src/snmalloc/ds/allocconfig.h → src/snmalloc/ds_core/allocconfig.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include "bits.h"
+#include "mitigations.h"
+
 namespace snmalloc
 {
   // 0 intermediate bits results in power of 2 small allocs. 1 intermediate

src/snmalloc/ds_core/defines.h

@@ -123,8 +123,11 @@ namespace snmalloc
   static constexpr bool Debug = true;
 #endif
 
-  // Forwards reference so that the platform can define how to handle errors.
+  // Forward references so that the platform can define how to handle
+  // errors and messages.  Definitions are provided in pal/pal.h once
+  // DefaultPal is known.
   [[noreturn]] SNMALLOC_COLD void error(const char* const str);
+  void message_impl(const char* const str);
 } // namespace snmalloc
 
 #define TOSTRING(expr) TOSTRING2(expr)
@@ -213,21 +216,21 @@ namespace snmalloc
 
 namespace snmalloc
 {
+  template<typename... Args>
+  SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...)
+  {}
+
   /**
-   * Forward declaration so that this can be called before the pal header is
-   * included.
+   * Forward declaration so that this can be called before helpers.h is
+   * included (e.g. in SNMALLOC_ASSERT macros expanded inside bits.h).
    */
   template<size_t BufferSize = 1024, typename... Args>
   [[noreturn]] inline void report_fatal_error(Args... args);
 
   /**
-   * Forward declaration so that this can be called before the pal header is
+   * Forward declaration so that this can be called before helpers.h is
    * included.
    */
   template<size_t BufferSize = 1024, typename... Args>
   inline void message(Args... args);
-
-  template<typename... Args>
-  SNMALLOC_FAST_PATH_INLINE void UNUSED(Args&&...)
-  {}
 } // namespace snmalloc

src/snmalloc/ds_core/ds_core.h

@@ -7,6 +7,7 @@
  * snmalloc.
  */
 
+#include "allocconfig.h"
 #include "bits.h"
 #include "cheri.h"
 #include "concept.h"
@@ -15,4 +16,5 @@
 #include "mitigations.h"
 #include "ptrwrap.h"
 #include "redblacktree.h"
+#include "sizeclassstatic.h"
 #include "tid.h"

src/snmalloc/ds_core/helpers.h

@@ -2,6 +2,7 @@
 
 #include "bits.h"
 #include "snmalloc/ds_core/defines.h"
+#include "snmalloc/ds_core/tid.h"
 #include "snmalloc/stl/array.h"
 #include "snmalloc/stl/type_traits.h"
 #include "snmalloc/stl/utility.h"
@@ -339,6 +340,38 @@ namespace snmalloc
     }
   };
 
+  /**
+   * Report a fatal error via a PAL-specific error reporting mechanism.  This
+   * takes a format string and a set of arguments.  The format string indicates
+   * the remaining arguments with "{}".  This could be extended later to
+   * support indexing fairly easily, if we ever want to localise these error
+   * messages.
+   *
+   * The following are supported as arguments:
+   *
+   *  - Characters (`char`), printed verbatim.
+   *  - Strings Literals (`const char*` or `const char[]`), printed verbatim.
+   *  - Raw pointers (void*), printed as hex strings.
+   *  - Integers (convertible to `size_t`), printed as hex strings.
+   *
+   *  These types should be sufficient for allocator-related error messages.
+   */
+  template<size_t BufferSize, typename... Args>
+  [[noreturn]] inline void report_fatal_error(Args... args)
+  {
+    MessageBuilder<BufferSize> msg{stl::forward<Args>(args)...};
+    error(msg.get_message());
+  }
+
+  template<size_t BufferSize, typename... Args>
+  inline void message(Args... args)
+  {
+    MessageBuilder<BufferSize> msg{stl::forward<Args>(args)...};
+    MessageBuilder<BufferSize> msg_tid{
+      "{}: {}", debug_get_tid(), msg.get_message()};
+    message_impl(msg_tid.get_message());
+  }
+
   /**
    * Convenience type that has no fields / methods.
    */

src/snmalloc/ds_core/redblacktree.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "snmalloc/ds_core/concept.h"
 #include "snmalloc/stl/array.h"
 
 #include <stddef.h>

src/snmalloc/ds_core/sizeclassstatic.h

@@ -0,0 +1,74 @@
+#pragma once
+
+#include "allocconfig.h"
+#include "bits.h"
+
+namespace snmalloc
+{
+  /**
+   * A wrapper type for small sizeclass indices.
+   *
+   * Implicitly converts TO size_t (for array indexing, comparisons, etc.)
+   * but does NOT implicitly convert FROM size_t — construction must be
+   * explicit via smallsizeclass_t(value).
+   */
+  struct smallsizeclass_t
+  {
+    size_t raw{0};
+
+    constexpr smallsizeclass_t() = default;
+
+    explicit constexpr smallsizeclass_t(size_t v) : raw(v) {}
+
+    /// Implicit conversion to size_t.
+    constexpr operator size_t() const
+    {
+      return raw;
+    }
+
+    /// Pre-increment.
+    constexpr smallsizeclass_t& operator++()
+    {
+      ++raw;
+      return *this;
+    }
+
+    /// Post-increment.
+    constexpr smallsizeclass_t operator++(int)
+    {
+      auto tmp = *this;
+      ++raw;
+      return tmp;
+    }
+  };
+
+  static constexpr smallsizeclass_t size_to_sizeclass_const(size_t size)
+  {
+    // Don't use sizeclasses that are not a multiple of the alignment.
+    // For example, 24 byte allocations can be
+    // problematic for some data due to alignment issues.
+    return smallsizeclass_t(
+      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(size));
+  }
+
+  constexpr size_t NUM_SMALL_SIZECLASSES =
+    size_t(size_to_sizeclass_const(MAX_SMALL_SIZECLASS_SIZE)) + 1;
+
+  static constexpr size_t sizeclass_to_size_const(smallsizeclass_t sc)
+  {
+    return bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(sc);
+  }
+
+  /**
+   * @brief Returns true if the size is a small sizeclass. Note that
+   * 0 is not considered a small sizeclass.
+   */
+  constexpr bool is_small_sizeclass(size_t size)
+  {
+    // Perform the - 1 on size, so that zero wraps around and ends up on
+    // slow path.
+    return (size - 1) <
+      sizeclass_to_size_const(smallsizeclass_t(NUM_SMALL_SIZECLASSES - 1));
+  }
+
+} // namespace snmalloc