Reclaim remote frees from disused allocators

src/snmalloc/ds/pool.h

@@ -80,8 +80,7 @@ namespace snmalloc
    *
    * The third template argument is a method to retrieve the actual PoolState.
    *
-   * For the pool of allocators, refer to the AllocPool alias defined in
-   * corealloc.h.
+   * The allocator pool provides its own construction helper and PoolState.
    *
    * For a pool of another type, it is recommended to leave the
    * third template argument with its default value. The SingletonPoolState
@@ -94,8 +93,53 @@ namespace snmalloc
     PoolState<T>& get_state() = SingletonPoolState<T>::pool>
   class Pool
   {
+    template<typename U>
+    static auto call_reinit(U* p, int) -> decltype(p->reinit())
+    {
+      return p->reinit();
+    }
+
+    template<typename U>
+    static void call_reinit(U*, long)
+    {}
+
+    template<typename U>
+    static auto call_deinit(U* p, int) -> decltype(p->deinit())
+    {
+      return p->deinit();
+    }
+
+    template<typename U>
+    static void call_deinit(U*, long)
+    {}
+
+    static void reinit(T* first)
+    {
+      T* item = first;
+      while (item != nullptr)
+      {
+        T* next = item->next.unsafe_ptr();
+        call_reinit(item, 0);
+        item = next;
+      }
+    }
+
+    static void deinit(T* first, T* last)
+    {
+      T* item = first;
+      while (true)
+      {
+        SNMALLOC_ASSERT(item != nullptr);
+        T* next = item->next.unsafe_ptr();
+        call_deinit(item, 0);
+        if (item == last)
+          break;
+        item = next;
+      }
+    }
+
   public:
-    static T* acquire()
+    static T* try_acquire_front()
     {
       PoolState<T>& pool = get_state();
 
@@ -115,10 +159,19 @@ namespace snmalloc
         }
       });
 
+      if (result != nullptr)
+        call_reinit(result, 0);
+      return result;
+    }
+
+    static T* acquire()
+    {
+      T* result = try_acquire_front();
       if (result != nullptr)
         return result;
 
       auto p = ConstructT::make();
+      PoolState<T>& pool = get_state();
 
       with(pool.lock, [&]() {
         p->list_next = pool.list;
@@ -155,6 +208,7 @@ namespace snmalloc
           pool.front = nullptr;
           pool.back = nullptr;
         });
+        reinit(result);
         return result;
       }
 
@@ -169,6 +223,7 @@ namespace snmalloc
     static void restore(T* first, T* last)
     {
       PoolState<T>& pool = get_state();
+      deinit(first, last);
       last->next = nullptr;
       with(pool.lock, [&]() {
         if (pool.front == nullptr)
@@ -192,6 +247,7 @@ namespace snmalloc
     static void restore_front(T* first, T* last)
     {
       PoolState<T>& pool = get_state();
+      deinit(first, last);
       last->next = nullptr;
 
       with(pool.lock, [&]() {

src/snmalloc/ds_core/ptrwrap.h

@@ -517,6 +517,20 @@ namespace snmalloc
       return CapPtr<T, bounds>::unsafe_from(
         this->unsafe_capptr.exchange(desired.unsafe_ptr(), order));
     }
+
+    SNMALLOC_FAST_PATH bool compare_exchange_strong(
+      CapPtr<T, bounds>& expected,
+      CapPtr<T, bounds> desired,
+      stl::MemoryOrder success_order,
+      stl::MemoryOrder failure_order) noexcept
+    {
+      auto raw_expected = expected.unsafe_ptr();
+      bool result = this->unsafe_capptr.compare_exchange_strong(
+        raw_expected, desired.unsafe_ptr(), success_order, failure_order);
+      if (!result)
+        expected = CapPtr<T, bounds>::unsafe_from(raw_expected);
+      return result;
+    }
   };
 
   namespace capptr

src/snmalloc/global/globalalloc.h

@@ -11,10 +11,10 @@ namespace snmalloc
     static_assert(
       Config_::Options.AllocIsPoolAllocated,
       "Global cleanup is available only for pool-allocated configurations");
+
     // Call this periodically to free and coalesce memory allocated by
     // allocators that are not currently in use by any thread.
-    // One atomic operation to extract the stack, another to restore it.
-    // Handling the message queue for each stack is non-atomic.
+    // Handling the message queue for each allocator is non-atomic.
     auto* first = AllocPool<Config_>::extract();
     auto* alloc = first;
 
@@ -43,6 +43,23 @@ namespace snmalloc
     static_assert(
       Config_::Options.AllocIsPoolAllocated,
       "Global status is available only for pool-allocated configurations");
+
+    // debug_is_empty() calls flush(), which requires an active message queue.
+    // Extract unused allocators to claim their queues for the whole check.
+    auto* first = AllocPool<Config_>::extract();
+    auto* last = first;
+    OnDestruct restore_free_allocators([&first, &last]() {
+      if (first != nullptr)
+        AllocPool<Config_>::restore(first, last);
+    });
+    while (last != nullptr)
+    {
+      auto* next = AllocPool<Config_>::extract(last);
+      if (next == nullptr)
+        break;
+      last = next;
+    }
+
     // This is a debugging function. It checks that all memory from all
     // allocators has been freed.
     auto* alloc = AllocPool<Config_>::iterate();

src/snmalloc/mem/allocpool_assistance.h

@@ -0,0 +1,144 @@
+#pragma once
+
+#include "../pal/pal_consts.h"
+#include "snmalloc/stl/atomic.h"
+
+#include <stdint.h>
+
+#ifndef SNMALLOC_ASSIST_IDLE_MS
+#  define SNMALLOC_ASSIST_IDLE_MS 1000
+#endif
+
+namespace snmalloc
+{
+  template<typename Config>
+  inline constexpr bool uses_inactive_queue_marker =
+    Config::Options.AllocIsPoolAllocated &&
+    pal_supports<Time, typename Config::Pal>;
+
+  /**
+   * State and policy for deciding when to assist a disused allocator.
+   */
+  template<typename Config>
+  class AllocPoolAssistance
+  {
+    /**
+     * Best-effort count of inactive allocators with pending remote frees.
+     * Queue-state transitions are authoritative; publication ordering may make
+     * this count temporarily non-positive.
+     */
+    SNMALLOC_REQUIRE_CONSTINIT
+    inline static stl::Atomic<int64_t> inactive_pending_count{0};
+
+    /**
+     * Time from which to measure the next assistance delay. Zero is an
+     * ordinary timestamp, not a sentinel.
+     */
+    SNMALLOC_REQUIRE_CONSTINIT
+    inline static stl::Atomic<uint64_t> idle_start_ms{0};
+
+    /**
+     * Incremented when pool acquisition claims a pending inactive allocator,
+     * which restarts the assistance delay.
+     */
+    SNMALLOC_REQUIRE_CONSTINIT
+    inline static stl::Atomic<uint64_t> deadline_reset_generation{0};
+
+    /**
+     * Last reset generation observed by the scheduling policy.
+     */
+    SNMALLOC_REQUIRE_CONSTINIT
+    inline static stl::Atomic<uint64_t> processed_deadline_reset_generation{0};
+
+    static bool process_deadline_reset(uint64_t sampled_time)
+    {
+      uint64_t processed =
+        processed_deadline_reset_generation.load(stl::memory_order_acquire);
+      uint64_t reset =
+        deadline_reset_generation.load(stl::memory_order_acquire);
+      if (processed == reset)
+        return false;
+
+      // This sample only restarts the delay; it does not reserve assistance.
+      idle_start_ms.store(sampled_time, stl::memory_order_relaxed);
+
+      // A failed CAS means another thread processed this generation. A newer
+      // generation remains different and will be processed by a later sample.
+      processed_deadline_reset_generation.compare_exchange_strong(
+        processed, reset, stl::memory_order_release, stl::memory_order_relaxed);
+      return true;
+    }
+
+  public:
+    /**
+     * Record an inactive allocator becoming responsible for pending remote
+     * frees.
+     */
+    static void pending_queue_added()
+    {
+      inactive_pending_count.fetch_add(1, stl::memory_order_relaxed);
+    }
+
+    /**
+     * Record a pending inactive allocator being claimed from the pool.
+     */
+    static void pending_queue_claimed()
+    {
+      inactive_pending_count.fetch_sub(1, stl::memory_order_relaxed);
+      deadline_reset_generation.fetch_add(1, stl::memory_order_release);
+    }
+
+    /**
+     * Debug-only observation of the best-effort number of inactive allocators
+     * with pending remote frees.
+     */
+    static int64_t debug_pending_count()
+    {
+      return inactive_pending_count.load(stl::memory_order_relaxed);
+    }
+
+    /**
+     * Return whether this sample should perform one assistance attempt.
+     *
+     * A successful return advances the shared pacing origin, ensuring that
+     * competing threads do not assist for the same interval.
+     */
+    [[nodiscard]] static bool should_assist(uint64_t sampled_time)
+    {
+      if constexpr (uses_inactive_queue_marker<Config>)
+      {
+        if (process_deadline_reset(sampled_time))
+          return false;
+
+        int64_t pending =
+          inactive_pending_count.load(stl::memory_order_acquire);
+        if (pending <= 0)
+          return false;
+
+        // More pending queues shorten the interval between assistance attempts.
+        uint64_t delay =
+          uint64_t{SNMALLOC_ASSIST_IDLE_MS} / static_cast<uint64_t>(pending);
+        if (delay == 0)
+          delay = 1;
+        uint64_t idle_start = idle_start_ms.load(stl::memory_order_relaxed);
+
+        // Unsigned subtraction handles clock wraparound. An older sample may
+        // cause an extra best-effort attempt, which is harmless.
+        if ((sampled_time - idle_start) < delay)
+          return false;
+
+        // Advancing the origin reserves this interval for the winning thread.
+        return idle_start_ms.compare_exchange_strong(
+          idle_start,
+          sampled_time,
+          stl::memory_order_relaxed,
+          stl::memory_order_relaxed);
+      }
+      else
+      {
+        UNUSED(sampled_time);
+        return false;
+      }
+    }
+  };
+} // namespace snmalloc