diff --git a/src/brpc/event_dispatcher.h b/src/brpc/event_dispatcher.h index 33f3537ebb..2a8c74ad43 100644 --- a/src/brpc/event_dispatcher.h +++ b/src/brpc/event_dispatcher.h @@ -20,6 +20,7 @@ #define BRPC_EVENT_DISPATCHER_H #include // DECLARE_bool +#include "butil/atomicops.h" // butil::atomic #include "butil/macros.h" // DISALLOW_COPY_AND_ASSIGN #include "bthread/types.h" // bthread_t, bthread_attr_t #include "brpc/versioned_ref_with_id.h" @@ -179,8 +180,8 @@ template friend class IOEvent; // The epoll/kqueue fd to watch events. int _event_dispatcher_fd; - // false unless Stop() is called. - volatile bool _stop; + // Set by Stop() to request shutdown. + butil::atomic _stop; // identifier of hosting bthread bthread_t _tid; diff --git a/src/brpc/event_dispatcher_epoll.cpp b/src/brpc/event_dispatcher_epoll.cpp index 1a3e74ef8e..1519f8d82b 100644 --- a/src/brpc/event_dispatcher_epoll.cpp +++ b/src/brpc/event_dispatcher_epoll.cpp @@ -94,11 +94,12 @@ int EventDispatcher::Start(const bthread_attr_t* thread_attr) { } bool EventDispatcher::Running() const { - return !_stop && _event_dispatcher_fd >= 0 && _tid != 0; + return !_stop.load(butil::memory_order_relaxed) && + _event_dispatcher_fd >= 0 && _tid != 0; } void EventDispatcher::Stop() { - _stop = true; + _stop.store(true, butil::memory_order_relaxed); if (_event_dispatcher_fd >= 0) { epoll_event evt = { EPOLLOUT, { nullptr } }; @@ -201,7 +202,7 @@ void* EventDispatcher::RunThis(void* arg) { } void EventDispatcher::Run() { - while (!_stop) { + while (!_stop.load(butil::memory_order_relaxed)) { epoll_event e[32]; #ifdef BRPC_ADDITIONAL_EPOLL // Performance downgrades in examples. @@ -212,10 +213,7 @@ void EventDispatcher::Run() { #else const int n = epoll_wait(_event_dispatcher_fd, e, ARRAY_SIZE(e), -1); #endif - if (_stop) { - // epoll_ctl/epoll_wait should have some sort of memory fencing - // guaranteeing that we(after epoll_wait) see _stop set before - // epoll_ctl. + if (_stop.load(butil::memory_order_relaxed)) { break; } if (n < 0) { diff --git a/src/brpc/event_dispatcher_kqueue.cpp b/src/brpc/event_dispatcher_kqueue.cpp index 8bec41aa8f..3e3100ea7b 100644 --- a/src/brpc/event_dispatcher_kqueue.cpp +++ b/src/brpc/event_dispatcher_kqueue.cpp @@ -95,11 +95,12 @@ int EventDispatcher::Start(const bthread_attr_t* thread_attr) { } bool EventDispatcher::Running() const { - return !_stop && _event_dispatcher_fd >= 0 && _tid != 0; + return !_stop.load(butil::memory_order_relaxed) && + _event_dispatcher_fd >= 0 && _tid != 0; } void EventDispatcher::Stop() { - _stop = true; + _stop.store(true, butil::memory_order_relaxed); if (_event_dispatcher_fd >= 0) { struct kevent kqueue_event; @@ -191,13 +192,10 @@ void* EventDispatcher::RunThis(void* arg) { } void EventDispatcher::Run() { - while (!_stop) { + while (!_stop.load(butil::memory_order_relaxed)) { struct kevent e[32]; int n = kevent(_event_dispatcher_fd, nullptr, 0, e, ARRAY_SIZE(e), nullptr); - if (_stop) { - // EV_SET/kevent should have some sort of memory fencing - // guaranteeing that we(after kevent) see _stop set before - // EV_SET + if (_stop.load(butil::memory_order_relaxed)) { break; } if (n < 0) { diff --git a/test/brpc_event_dispatcher_unittest.cpp b/test/brpc_event_dispatcher_unittest.cpp index 1825fe296d..c3795715a9 100644 --- a/test/brpc_event_dispatcher_unittest.cpp +++ b/test/brpc_event_dispatcher_unittest.cpp @@ -130,6 +130,51 @@ TEST_F(EventDispatcherTest, has_epollrdhup) { LOG(INFO) << brpc::has_epollrdhup; } +TEST_F(EventDispatcherTest, start_stop_updates_running_state) { + brpc::EventDispatcher dispatcher; + ASSERT_EQ(0, dispatcher.Start(nullptr)); + ASSERT_TRUE(dispatcher.Running()); + dispatcher.Stop(); + ASSERT_FALSE(dispatcher.Running()); + dispatcher.Join(); + ASSERT_FALSE(dispatcher.Running()); +} + +struct DispatcherStopRaceArgs { + brpc::EventDispatcher* dispatcher; + butil::atomic call_running; + + DispatcherStopRaceArgs() + : dispatcher(nullptr) + , call_running(false) {} +}; + +void* call_dispatcher_running(void* arg) { + DispatcherStopRaceArgs* args = static_cast(arg); + while (!args->call_running.load(butil::memory_order_relaxed)) { + } + args->dispatcher->Running(); + return nullptr; +} + +// The relaxed handoff guarantees that Running() is called after Stop() without +// creating a happens-before edge. Under TSAN, this reports the Running()/Stop() +// race when _stop is non-atomic. +TEST_F(EventDispatcherTest, running_is_safe_after_stop) { + brpc::EventDispatcher dispatcher; + ASSERT_EQ(0, dispatcher.Start(nullptr)); + + DispatcherStopRaceArgs args; + args.dispatcher = &dispatcher; + pthread_t tid; + ASSERT_EQ(0, pthread_create(&tid, nullptr, call_dispatcher_running, + &args)); + dispatcher.Stop(); + args.call_running.store(true, butil::memory_order_relaxed); + ASSERT_EQ(0, pthread_join(tid, nullptr)); + dispatcher.Join(); +} + TEST_F(EventDispatcherTest, versioned_ref) { butil::atomic versioned_ref(2); versioned_ref.fetch_add(brpc::MakeVRef(0, -1), @@ -647,7 +692,7 @@ void* stop_and_join(void* arg) { StopArgs* args = static_cast(arg); args->server->Stop(0); args->server->Join(); - args->done.store(true, butil::memory_order_release); + args->done.store(true, butil::memory_order_relaxed); return nullptr; } @@ -705,11 +750,11 @@ TEST_F(EventDispatcherTest, server_with_fewer_threads_than_dispatchers_can_quit) pthread_t th; ASSERT_EQ(0, pthread_create(&th, nullptr, stop_and_join, &args)); int64_t deadline_us = butil::gettimeofday_us() + 10 * 1000000L; - while (!args.done.load(butil::memory_order_acquire) && + while (!args.done.load(butil::memory_order_relaxed) && butil::gettimeofday_us() < deadline_us) { usleep(10000); } - bool done = args.done.load(butil::memory_order_acquire); + bool done = args.done.load(butil::memory_order_relaxed); if (!done) { EXPECT_TRUE(done) << "Server::Stop()/Join() did not finish in 10s, " "no worker available?";