Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions src/brpc/rdma/rdma_endpoint.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1954,6 +1954,12 @@ void RdmaEndpoint::GlobalRelease() {

std::vector<RdmaEndpoint::PollerGroup> RdmaEndpoint::_poller_groups;

#if defined(UNIT_TEST)
// Inject poller creation failures without requiring an RDMA device.
decltype(&bthread_start_background) g_start_rdma_poller_for_test =
bthread_start_background;
#endif
Comment on lines +1957 to +1961

int RdmaEndpoint::PollingModeInitialize(bthread_tag_t tag,
std::function<void()> callback,
std::function<void()> init_fn,
Expand Down Expand Up @@ -2012,20 +2018,32 @@ int RdmaEndpoint::PollingModeInitialize(bthread_tag_t tag,

return nullptr;
};
for (int i = 0; i < FLAGS_rdma_poller_num; ++i) {
auto args = new FnArgs{&pollers[i], &running};
for (size_t i = 0; i < pollers.size(); ++i) {
std::unique_ptr<FnArgs> args(new FnArgs{&pollers[i], &running});
auto attr = FLAGS_rdma_disable_bthread ? BTHREAD_ATTR_PTHREAD
: BTHREAD_ATTR_NORMAL;
attr.tag = tag;
bthread_attr_set_name(&attr, "RdmaPolling");
pollers[i].callback = callback;
pollers[i].init_fn = init_fn;
pollers[i].release_fn = release_fn;
auto rc = bthread_start_background(&pollers[i].tid, &attr, fn, args);
#if defined(UNIT_TEST)
const int rc = g_start_rdma_poller_for_test(
#else
const int rc = bthread_start_background(
#endif
&pollers[i].tid, &attr, fn, args.get());
if (rc != 0) {
LOG(ERROR) << "Fail to start rdma polling bthread";
running.store(false, std::memory_order_relaxed);
for (size_t j = 0; j < i; ++j) {
bthread_join(pollers[j].tid, nullptr);
pollers[j].tid = INVALID_BTHREAD;

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Setting running to false before the old poller exits allows another initialization with the same tag to pass the entry CAS. Specifically, after A starts the first poller, creation of the second fails and A enters join. If B then initializes another Channel and sets running back to true, the old poller, which has not yet observed the stop signal, will continue running, and A's initialization will remain blocked. Initializations of different Channels share this PollerGroup, and the call chain has no serialization protection. Consider separating the stop signal from the initialization state to prevent the next batch of tasks from starting before rollback completes, and add a barrier-controlled concurrency test.

}
LOG(ERROR) << "Fail to start rdma polling bthread: " << berror(rc);
errno = rc;
return -1;
}
args.release();
}
return 0;
}
Expand All @@ -2038,8 +2056,11 @@ void RdmaEndpoint::PollingModeRelease(bthread_tag_t tag) {
auto& pollers = group.pollers;
auto& running = group.running;
running.store(false, std::memory_order_relaxed);
for (int i = 0; i < FLAGS_rdma_poller_num; ++i) {
bthread_join(pollers[i].tid, nullptr);
for (auto& poller : pollers) {
if (poller.tid != INVALID_BTHREAD) {
bthread_join(poller.tid, nullptr);
poller.tid = INVALID_BTHREAD;
}
}
}

Expand Down
123 changes: 123 additions & 0 deletions test/brpc_rdma_unittest.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#if BRPC_WITH_RDMA
#include <errno.h>
#include <unistd.h>
#include <atomic>
#include <functional>
#include <vector>
#include <google/protobuf/descriptor.h>
Expand All @@ -32,6 +33,7 @@
#include "butil/sys_byteorder.h"
#include "butil/time.h"
#include "butil/files/temp_file.h"
#include "bthread/bthread.h"
#include "brpc/acceptor.h"
#include "brpc/channel.h"
#include "brpc/controller.h"
Expand Down Expand Up @@ -70,6 +72,8 @@ DECLARE_bool(rdma_trace_verbose);
DECLARE_int32(rdma_memory_pool_max_regions);
DECLARE_int32(rdma_client_handshake_version);
DECLARE_bool(rdma_ece);
DECLARE_bool(rdma_poller_yield);
extern decltype(&bthread_start_background) g_start_rdma_poller_for_test;

extern ibv_cq* (*IbvCreateCq)(ibv_context*, int, void*, ibv_comp_channel*, int);
extern int (*IbvDestroyCq)(ibv_cq*);
Expand Down Expand Up @@ -3569,6 +3573,125 @@ TEST(RdmaInlineDataTest, should_post_inline) {
EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0));
}

// Poller lifecycle tests do not need any RDMA queues or hardware.
static int g_poller_start_count = 0;
static int g_poller_fail_at = -1;

static int StartRdmaPollerForTest(bthread_t* tid, const bthread_attr_t* attr,
void* (*fn)(void*), void* arg) {
if (g_poller_start_count++ == g_poller_fail_at) {
return EAGAIN;
}
return bthread_start_background(tid, attr, fn, arg);
}

class RdmaPollingTest : public ::testing::TestWithParam<int> {
protected:
void SetUp() override {
_saved_use_polling = rdma::FLAGS_rdma_use_polling;
_saved_poller_num = rdma::FLAGS_rdma_poller_num;
_saved_poller_yield = rdma::FLAGS_rdma_poller_yield;
_saved_disable_bthread = rdma::FLAGS_rdma_disable_bthread;
_saved_start_poller = rdma::g_start_rdma_poller_for_test;
rdma::FLAGS_rdma_use_polling = true;
rdma::FLAGS_rdma_poller_num = 3;
rdma::FLAGS_rdma_poller_yield = true;
_saved_groups.swap(rdma::RdmaEndpoint::_poller_groups);
rdma::RdmaEndpoint::_poller_groups =
std::vector<rdma::RdmaEndpoint::PollerGroup>(1);
rdma::g_start_rdma_poller_for_test = StartRdmaPollerForTest;
}

void TearDown() override {
rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
rdma::g_start_rdma_poller_for_test = _saved_start_poller;
_saved_groups.swap(rdma::RdmaEndpoint::_poller_groups);
rdma::FLAGS_rdma_use_polling = _saved_use_polling;
rdma::FLAGS_rdma_poller_num = _saved_poller_num;
rdma::FLAGS_rdma_poller_yield = _saved_poller_yield;
rdma::FLAGS_rdma_disable_bthread = _saved_disable_bthread;
g_poller_fail_at = -1;
}

void ExpectStopped() {
const auto& group =
rdma::RdmaEndpoint::_poller_groups[BTHREAD_TAG_DEFAULT];
EXPECT_FALSE(group.running.load());
for (const auto& poller : group.pollers) {
EXPECT_EQ(INVALID_BTHREAD, poller.tid);
}
}

std::atomic<int> _initialized{0};
std::atomic<int> _released{0};
std::atomic<int> _callbacks{0};

private:
bool _saved_use_polling;
int _saved_poller_num;
bool _saved_poller_yield;
bool _saved_disable_bthread;
decltype(&bthread_start_background) _saved_start_poller;
std::vector<rdma::RdmaEndpoint::PollerGroup> _saved_groups;
};

TEST_P(RdmaPollingTest, failed_creation_can_retry) {
// Exercise both the bthread and pthread-backed pollers.
for (bool disable_bthread : {false, true}) {
SCOPED_TRACE(disable_bthread);
rdma::FLAGS_rdma_disable_bthread = disable_bthread;
_initialized.store(0);
_released.store(0);
_callbacks.store(0);
g_poller_start_count = 0;
g_poller_fail_at = GetParam();
auto callback = [this] { _callbacks.fetch_add(1); };
auto init_fn = [this] { _initialized.fetch_add(1); };
// Cleanup must preserve the creation error even if the callback
// changes errno.
auto release_fn = [this] {
_released.fetch_add(1);
errno = EINVAL;
};
ASSERT_EQ(-1, rdma::RdmaEndpoint::PollingModeInitialize(
BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn));
EXPECT_EQ(EAGAIN, errno);
EXPECT_EQ(GetParam() + 1, g_poller_start_count);
ExpectStopped();
EXPECT_EQ(GetParam(), _initialized.load());
EXPECT_EQ(GetParam(), _released.load());

// Release is safe even after a partially initialized group rolls back.
rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
ExpectStopped();
EXPECT_EQ(GetParam(), _released.load());

g_poller_start_count = 0;
g_poller_fail_at = -1;
ASSERT_EQ(0, rdma::RdmaEndpoint::PollingModeInitialize(
BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn));
ASSERT_TRUE(WaitUntil([this] {
return _initialized.load() == GetParam() + 3 &&
_callbacks.load() > 0;
}));
EXPECT_EQ(3, g_poller_start_count);
EXPECT_TRUE(rdma::RdmaEndpoint::_poller_groups[BTHREAD_TAG_DEFAULT]
.running.load());
EXPECT_EQ(0, rdma::RdmaEndpoint::PollingModeInitialize(
BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn));
EXPECT_EQ(3, g_poller_start_count);

rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
ExpectStopped();
EXPECT_EQ(GetParam() + 3, _released.load());
rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT);
EXPECT_EQ(GetParam() + 3, _released.load());
}
}

INSTANTIATE_TEST_SUITE_P(FailureAtEachPoller, RdmaPollingTest,
::testing::Values(0, 1, 2));

#endif // if BRPC_WITH_RDMA

int main(int argc, char* argv[]) {
Expand Down
Loading