Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/brpc/event_dispatcher.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
#define BRPC_EVENT_DISPATCHER_H

#include <gflags/gflags_declare.h> // DECLARE_bool
#include "butil/atomicops.h" // butil::atomic
#include "butil/macros.h" // DISALLOW_COPY_AND_ASSIGN
#include "bthread/types.h" // bthread_t, bthread_attr_t
#include "brpc/versioned_ref_with_id.h"
Expand Down Expand Up @@ -179,8 +180,8 @@ template <typename T> friend class IOEvent;
// The epoll/kqueue fd to watch events.
int _event_dispatcher_fd;

// false unless Stop() is called.
volatile bool _stop;
// Set by Stop() to request shutdown.
butil::atomic<bool> _stop;

// identifier of hosting bthread
bthread_t _tid;
Expand Down
12 changes: 5 additions & 7 deletions src/brpc/event_dispatcher_epoll.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,12 @@ int EventDispatcher::Start(const bthread_attr_t* thread_attr) {
}

bool EventDispatcher::Running() const {
return !_stop && _event_dispatcher_fd >= 0 && _tid != 0;
return !_stop.load(butil::memory_order_relaxed) &&
_event_dispatcher_fd >= 0 && _tid != 0;
}

void EventDispatcher::Stop() {
_stop = true;
_stop.store(true, butil::memory_order_relaxed);

if (_event_dispatcher_fd >= 0) {
epoll_event evt = { EPOLLOUT, { nullptr } };
Expand Down Expand Up @@ -201,7 +202,7 @@ void* EventDispatcher::RunThis(void* arg) {
}

void EventDispatcher::Run() {
while (!_stop) {
while (!_stop.load(butil::memory_order_relaxed)) {
epoll_event e[32];
#ifdef BRPC_ADDITIONAL_EPOLL
// Performance downgrades in examples.
Expand All @@ -212,10 +213,7 @@ void EventDispatcher::Run() {
#else
const int n = epoll_wait(_event_dispatcher_fd, e, ARRAY_SIZE(e), -1);
#endif
if (_stop) {
// epoll_ctl/epoll_wait should have some sort of memory fencing
// guaranteeing that we(after epoll_wait) see _stop set before
// epoll_ctl.
if (_stop.load(butil::memory_order_relaxed)) {
break;
}
if (n < 0) {
Expand Down
12 changes: 5 additions & 7 deletions src/brpc/event_dispatcher_kqueue.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -95,11 +95,12 @@ int EventDispatcher::Start(const bthread_attr_t* thread_attr) {
}

bool EventDispatcher::Running() const {
return !_stop && _event_dispatcher_fd >= 0 && _tid != 0;
return !_stop.load(butil::memory_order_relaxed) &&
_event_dispatcher_fd >= 0 && _tid != 0;
}

void EventDispatcher::Stop() {
_stop = true;
_stop.store(true, butil::memory_order_relaxed);

if (_event_dispatcher_fd >= 0) {
struct kevent kqueue_event;
Expand Down Expand Up @@ -191,13 +192,10 @@ void* EventDispatcher::RunThis(void* arg) {
}

void EventDispatcher::Run() {
while (!_stop) {
while (!_stop.load(butil::memory_order_relaxed)) {
struct kevent e[32];
int n = kevent(_event_dispatcher_fd, nullptr, 0, e, ARRAY_SIZE(e), nullptr);
if (_stop) {
// EV_SET/kevent should have some sort of memory fencing
// guaranteeing that we(after kevent) see _stop set before
// EV_SET
if (_stop.load(butil::memory_order_relaxed)) {
break;
}
if (n < 0) {
Expand Down
51 changes: 48 additions & 3 deletions test/brpc_event_dispatcher_unittest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,51 @@ TEST_F(EventDispatcherTest, has_epollrdhup) {
LOG(INFO) << brpc::has_epollrdhup;
}

TEST_F(EventDispatcherTest, start_stop_updates_running_state) {
brpc::EventDispatcher dispatcher;
ASSERT_EQ(0, dispatcher.Start(nullptr));
ASSERT_TRUE(dispatcher.Running());
dispatcher.Stop();
ASSERT_FALSE(dispatcher.Running());
dispatcher.Join();
ASSERT_FALSE(dispatcher.Running());
}

struct DispatcherStopRaceArgs {
brpc::EventDispatcher* dispatcher;
butil::atomic<bool> call_running;

DispatcherStopRaceArgs()
: dispatcher(nullptr)
, call_running(false) {}
};

void* call_dispatcher_running(void* arg) {
DispatcherStopRaceArgs* args = static_cast<DispatcherStopRaceArgs*>(arg);
while (!args->call_running.load(butil::memory_order_relaxed)) {
}
args->dispatcher->Running();
return nullptr;
}

// The relaxed handoff guarantees that Running() is called after Stop() without
// creating a happens-before edge. Under TSAN, this reports the Running()/Stop()
// race when _stop is non-atomic.
TEST_F(EventDispatcherTest, running_is_safe_after_stop) {
brpc::EventDispatcher dispatcher;
ASSERT_EQ(0, dispatcher.Start(nullptr));

DispatcherStopRaceArgs args;
args.dispatcher = &dispatcher;
pthread_t tid;
ASSERT_EQ(0, pthread_create(&tid, nullptr, call_dispatcher_running,
&args));
dispatcher.Stop();
args.call_running.store(true, butil::memory_order_relaxed);
ASSERT_EQ(0, pthread_join(tid, nullptr));
dispatcher.Join();
}

TEST_F(EventDispatcherTest, versioned_ref) {
butil::atomic<uint64_t> versioned_ref(2);
versioned_ref.fetch_add(brpc::MakeVRef(0, -1),
Expand Down Expand Up @@ -647,7 +692,7 @@ void* stop_and_join(void* arg) {
StopArgs* args = static_cast<StopArgs*>(arg);
args->server->Stop(0);
args->server->Join();
args->done.store(true, butil::memory_order_release);
args->done.store(true, butil::memory_order_relaxed);
return nullptr;
}

Expand Down Expand Up @@ -705,11 +750,11 @@ TEST_F(EventDispatcherTest, server_with_fewer_threads_than_dispatchers_can_quit)
pthread_t th;
ASSERT_EQ(0, pthread_create(&th, nullptr, stop_and_join, &args));
int64_t deadline_us = butil::gettimeofday_us() + 10 * 1000000L;
while (!args.done.load(butil::memory_order_acquire) &&
while (!args.done.load(butil::memory_order_relaxed) &&
butil::gettimeofday_us() < deadline_us) {
usleep(10000);
}
bool done = args.done.load(butil::memory_order_acquire);
bool done = args.done.load(butil::memory_order_relaxed);
if (!done) {
EXPECT_TRUE(done) << "Server::Stop()/Join() did not finish in 10s, "
"no worker available?";
Expand Down
Loading