From cf9d8598ef8ec1a07ff728f268627facc10befa0 Mon Sep 17 00:00:00 2001 From: Steve Gerbino Date: Wed, 11 Nov 2026 23:15:38 +0100 Subject: [PATCH] perf: run handlers queued behind the reactor first while that pays off When the reactor sentinel reached the front with handlers still queued, the scheduler polled the reactor without blocking before running them. Those handlers usually start the next I/O themselves, such as a coroutine an I/O completion woke, so running them first can save a reactor pass. The sentinel now moves behind the queued handlers once and polls on its next turn regardless, so handlers that keep reposting still cannot starve I/O. Waiting does not always pay: when the handlers drain the queue, the deferred turn has nothing left to overlap with, and where loopback data arrives later their I/O parks instead of completing. reactor_defer_policy scores each deferral by whether its turn still found handlers queued, and stops deferring while deferrals that empty the queue are frequent. Turns that poll instead let it try again. restart() forgets a pending deferral and the score, so a run stopped with the sentinel moved back doesn't cost the next run its own. poll() and poll_one() are unchanged. --- .../detail/reactor/reactor_defer_policy.hpp | 90 ++++++++++++ .../detail/reactor/reactor_scheduler.hpp | 32 +++++ test/unit/fault/kqueue_faults.cpp | 19 ++- test/unit/fault/reactor_faults.hpp | 73 ++++++++++ test/unit/reactor_defer_policy.cpp | 130 ++++++++++++++++++ 5 files changed, 333 insertions(+), 11 deletions(-) create mode 100644 include/boost/corosio/native/detail/reactor/reactor_defer_policy.hpp create mode 100644 test/unit/reactor_defer_policy.cpp diff --git a/include/boost/corosio/native/detail/reactor/reactor_defer_policy.hpp b/include/boost/corosio/native/detail/reactor/reactor_defer_policy.hpp new file mode 100644 index 000000000..e02904542 --- /dev/null +++ b/include/boost/corosio/native/detail/reactor/reactor_defer_policy.hpp @@ -0,0 +1,90 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_DEFER_POLICY_HPP +#define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_DEFER_POLICY_HPP + +namespace boost::corosio::detail { + +/** Decide whether the reactor sentinel waits behind queued handlers. + + When the sentinel reaches the front of the queue with handlers still + queued, running those handlers first usually saves a reactor pass: + they tend to start the next I/O themselves. That bet loses when the + handlers drain the queue, so the deferred turn has no other work to + overlap with and a poll that would have found completed I/O comes too + late. The policy scores each deferral by what the deferred turn found + and stops deferring while losses are frequent. + + A deferred turn that found the queue empty costs more than one that + found handlers repays, so deferring continues only while fewer than + about one deferral in `miss_cost + 1` empties the queue. Each turn + that polls instead of deferring lets the score decay, so a workload + that changes is tried again. + + @par Thread Safety + Distinct objects: Safe. + Shared objects: Unsafe. The scheduler calls it under its own mutex. +*/ +class reactor_defer_policy +{ + int score_ = 0; + +public: + /// Score added by a deferred turn that found the queue empty. + static constexpr int miss_cost = 8; + + /// Score at or above which the sentinel polls instead of deferring. + static constexpr int threshold = 8; + + /// Highest score, which bounds how long deferring stays off. + static constexpr int max_score = 64; + + /// Check whether the sentinel may wait behind queued handlers. + bool may_defer() const noexcept + { + return score_ < threshold; + } + + /** Record what a deferred turn of the sentinel found. + + @param handlers_queued `true` if handlers were still queued, so + the deferral cost nothing. + */ + void on_deferred_turn(bool handlers_queued) noexcept + { + if (handlers_queued) + { + if (score_ > 0) + --score_; + } + else + { + score_ = + score_ > max_score - miss_cost ? max_score : score_ + miss_cost; + } + } + + /// Record a turn that polled because deferring was off. + void on_polled_instead() noexcept + { + if (score_ > 0) + --score_; + } + + /// Forget the history, as for a fresh run. + void reset() noexcept + { + score_ = 0; + } +}; + +} // namespace boost::corosio::detail + +#endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_DEFER_POLICY_HPP diff --git a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp index 41612f02a..0d0ca6581 100644 --- a/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp +++ b/include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -333,6 +334,11 @@ class reactor_scheduler : public scheduler std::atomic stopped_{false}; mutable std::atomic task_running_{false}; mutable bool task_interrupted_ = false; + // Set while the sentinel waits behind queued handlers (guarded by + // mutex_). + bool task_deferred_ = false; + // Whether waiting is paying off (guarded by mutex_). + reactor_defer_policy defer_policy_; // Descriptors retired during the running reactor pass (guarded by // mutex_). @@ -635,6 +641,11 @@ reactor_scheduler::stopped() const noexcept inline void reactor_scheduler::restart() { + // A run can end with the sentinel marked deferred; clearing the + // mark lets the next run defer it again. + lock_type lock(mutex_); + task_deferred_ = false; + defer_policy_.reset(); stopped_.store(false, std::memory_order_release); } @@ -1042,6 +1053,27 @@ reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx) { bool more_handlers = !completed_ops_.empty(); + // Handlers queued here usually start the next I/O + // themselves, so running them first can save a reactor pass. + // Defer at most once in a row so reposting work cannot + // starve I/O. poll() and poll_one() don't defer: the reactor + // turn they would skip may be their last before returning. + if (task_deferred_) + { + task_deferred_ = false; + defer_policy_.on_deferred_turn(more_handlers); + } + else if (more_handlers && timeout_us != 0) + { + if (defer_policy_.may_defer()) + { + task_deferred_ = true; + completed_ops_.push(&task_op_); + continue; + } + defer_policy_.on_polled_instead(); + } + if (!more_handlers && outstanding_work_.load(std::memory_order_acquire) == 0) { diff --git a/test/unit/fault/kqueue_faults.cpp b/test/unit/fault/kqueue_faults.cpp index 19da85ba5..77c36aa0e 100644 --- a/test/unit/fault/kqueue_faults.cpp +++ b/test/unit/fault/kqueue_faults.cpp @@ -329,14 +329,12 @@ struct kqueue_faults void testRunLoopFaults() { - // Nothing calls kevent between the constructor and the run - // loop's first wait: posting the task does not interrupt a - // reactor that is not running yet - // (reactor_scheduler::wake_one_thread_and_unlock), so nth 1 is - // that wait. + // The run loop runs the posted task before its first wait, and + // arming the delay triggers the user event, so nth 1 is that + // trigger and nth 2 is the wait. { io_context ioc(kqueue); - fault_scope f(sys::kevent, EINTR); + fault_scope f(sys::kevent, EINTR, 2); bool done = false; auto body = [&]() -> capy::task<> { std::ignore = @@ -352,7 +350,7 @@ struct kqueue_faults // Any other errno leaves run() through an exception // (kqueue_scheduler::run_task). io_context ioc(kqueue); - fault_scope f(sys::kevent, EBADF); + fault_scope f(sys::kevent, EBADF, 2); auto body = [&]() -> capy::task<> { std::ignore = co_await corosio::delay(std::chrono::milliseconds(1)); @@ -377,10 +375,9 @@ struct kqueue_faults { io_context ioc(kqueue); { - // stop() is the one path that reaches NOTE_TRIGGER without - // a reactor thread (reactor_scheduler::stop), and it - // interrupts only on the transition, so each stop below is - // one kevent. + // stop() reaches NOTE_TRIGGER with no other kevent call in + // between, and it interrupts only on the transition, so each + // stop below is one kevent. fault_scope first(sys::kevent, EIO, 1); fault_scope second(sys::kevent, EIO, 2); ioc.stop(); diff --git a/test/unit/fault/reactor_faults.hpp b/test/unit/fault/reactor_faults.hpp index d4fab93f7..4479f808c 100644 --- a/test/unit/fault/reactor_faults.hpp +++ b/test/unit/fault/reactor_faults.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -84,6 +85,9 @@ struct reactor_common_faults static constexpr sys spec_write = sys::write; #endif static constexpr sys vec_write = is_kqueue ? sys::writev : sys::sendmsg; + static constexpr sys wait_sys = is_select ? sys::select + : is_kqueue ? sys::kevent + : sys::epoll_wait; static endpoint loopback() { @@ -1034,6 +1038,73 @@ struct reactor_common_faults #endif } + // A handler that an I/O completion posts runs before the reactor is + // polled again, since that handler may start the next I/O itself. + void testPostedWakeRunsBeforeNextPoll() + { + io_context ioc(Backend); + auto [a, b] = test::must(test::make_socket_pair(ioc)); + capy::async_waker waker; + unsigned at_read = 0; + unsigned at_wake = 0; + // Counts this thread's calls to the wait symbol (every kevent on + // kqueue); the one it would fail is never reached. + fault_scope f(wait_sys, EINTR, 1u << 30); + + auto waiter = [&]() -> capy::task<> { + auto [ec] = co_await waker.wait(); + BOOST_TEST(!ec); + at_wake = f.count(); + }; + auto reader = [&]() -> capy::task<> { + char buf[1]; + auto [ec, n] = co_await b.read_some(capy::mutable_buffer(buf, 1)); + BOOST_TEST(!ec); + at_read = f.count(); + waker.wake(); + }; + auto writer = [&]() -> capy::task<> { + char const c = 'x'; + auto [ec, n] = co_await a.write_some(capy::const_buffer(&c, 1)); + BOOST_TEST(!ec); + }; + capy::run_async(ioc.get_executor())(waiter()); + capy::run_async(ioc.get_executor())(reader()); + capy::run_async(ioc.get_executor())(writer()); + ioc.run(); + + // The read parked, so a poll completed it. + BOOST_TEST(at_read > 0); + BOOST_TEST_EQ(at_wake, at_read); + } + + // A run stopped while the sentinel was deferred leaves the next run + // its own deferral, so a posted handler still runs before a poll. + void testRestartRenewsDeferral() + { + io_context ioc(Backend); + auto stopper = [&]() -> capy::task<> { + ioc.stop(); + co_return; + }; + capy::run_async(ioc.get_executor())(stopper()); + ioc.run(); + ioc.restart(); + + // Counts this thread's calls to the wait symbol (every kevent on + // kqueue); the one it would fail is never reached. + fault_scope f(wait_sys, EINTR, 1u << 30); + unsigned at_run = 1; + auto probe = [&]() -> capy::task<> { + at_run = f.count(); + co_return; + }; + capy::run_async(ioc.get_executor())(probe()); + ioc.run(); + + BOOST_TEST_EQ(at_run, 0u); + } + // Accepted and assigned sockets learn their family from the // addresses the library already reads, so option calls never ask. void testAdoptedSocketsKnowFamily() @@ -1352,6 +1423,8 @@ struct reactor_common_faults testSocketFamilyProbeFails(); testOpenedSocketKnowsFamily(); testAdoptedSocketsKnowFamily(); + testPostedWakeRunsBeforeNextPoll(); + testRestartRenewsDeferral(); testDatagramFails(); } }; diff --git a/test/unit/reactor_defer_policy.cpp b/test/unit/reactor_defer_policy.cpp new file mode 100644 index 000000000..53c9adf75 --- /dev/null +++ b/test/unit/reactor_defer_policy.cpp @@ -0,0 +1,130 @@ +// +// Copyright (c) 2026 Steve Gerbino +// +// Distributed under the Boost Software License, Version 1.0. (See accompanying +// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) +// +// Official repository: https://github.com/cppalliance/corosio +// + +#include + +#include "test_suite.hpp" + +using boost::corosio::detail::reactor_defer_policy; + +namespace { + +// Drive the policy as the scheduler does for `turns` sentinel turns that +// find handlers queued; `queued_on_deferred_turn(i)` says whether the i-th +// deferral's turn still found handlers. Returns the deferrals made. +template +int +drive(reactor_defer_policy& p, int turns, F queued_on_deferred_turn) +{ + int deferrals = 0; + for (int t = 0; t < turns; ++t) + { + if (p.may_defer()) + p.on_deferred_turn(queued_on_deferred_turn(deferrals++)); + else + p.on_polled_instead(); + } + return deferrals; +} + +} // namespace + +struct reactor_defer_policy_test +{ + void testFreshPolicyDefers() + { + reactor_defer_policy p; + BOOST_TEST(p.may_defer()); + } + + void testDeferralsThatPayKeepDeferring() + { + reactor_defer_policy p; + int const n = drive(p, 1000, [](int) { return true; }); + BOOST_TEST_EQ(n, 1000); + BOOST_TEST(p.may_defer()); + } + + void testEmptiedQueueStopsDeferring() + { + reactor_defer_policy p; + p.on_deferred_turn(false); + BOOST_TEST(!p.may_defer()); + } + + void testPollingRetriesDeferral() + { + reactor_defer_policy p; + p.on_deferred_turn(false); + for (int i = 0; i < reactor_defer_policy::miss_cost - + reactor_defer_policy::threshold + 1; + ++i) + p.on_polled_instead(); + BOOST_TEST(p.may_defer()); + } + + // Losing every time still tries again, but only on about one turn in + // `miss_cost + 1`. + void testConstantLossesBackOff() + { + reactor_defer_policy p; + int const turns = 9000; + int const n = drive(p, turns, [](int) { return false; }); + BOOST_TEST(n > 0); + BOOST_TEST(n <= turns / (reactor_defer_policy::miss_cost + 1) + 2); + } + + // A loss in fewer than one deferral in `miss_cost + 1` costs only the + // turn after it; every other turn still defers. + void testRareLossesKeepDeferring() + { + reactor_defer_policy p; + int const period = reactor_defer_policy::miss_cost + 2; + int const turns = 1000; + int const n = drive(p, turns, [&](int i) { return i % period != 0; }); + int const losses = (n + period - 1) / period; + BOOST_TEST_EQ(turns - n, losses); + } + + // However long a workload loses, a later change is tried within a + // bounded number of polls. + void testLossesAreBounded() + { + reactor_defer_policy p; + for (int i = 0; i < 1000; ++i) + p.on_deferred_turn(false); + for (int i = 0; i < reactor_defer_policy::max_score - + reactor_defer_policy::threshold + 1; + ++i) + p.on_polled_instead(); + BOOST_TEST(p.may_defer()); + } + + void testResetForgetsLosses() + { + reactor_defer_policy p; + p.on_deferred_turn(false); + p.reset(); + BOOST_TEST(p.may_defer()); + } + + void run() + { + testFreshPolicyDefers(); + testDeferralsThatPayKeepDeferring(); + testEmptiedQueueStopsDeferring(); + testPollingRetriesDeferral(); + testConstantLossesBackOff(); + testRareLossesKeepDeferring(); + testLossesAreBounded(); + testResetForgetsLosses(); + } +}; + +TEST_SUITE(reactor_defer_policy_test, "boost.corosio.reactor_defer_policy");