diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 110279b701..0bf83178b8 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -1954,6 +1954,12 @@ void RdmaEndpoint::GlobalRelease() { std::vector RdmaEndpoint::_poller_groups; +#if defined(UNIT_TEST) +// Inject poller creation failures without requiring an RDMA device. +decltype(&bthread_start_background) g_start_rdma_poller_for_test = + bthread_start_background; +#endif + int RdmaEndpoint::PollingModeInitialize(bthread_tag_t tag, std::function callback, std::function init_fn, @@ -2012,8 +2018,8 @@ int RdmaEndpoint::PollingModeInitialize(bthread_tag_t tag, return nullptr; }; - for (int i = 0; i < FLAGS_rdma_poller_num; ++i) { - auto args = new FnArgs{&pollers[i], &running}; + for (size_t i = 0; i < pollers.size(); ++i) { + std::unique_ptr args(new FnArgs{&pollers[i], &running}); auto attr = FLAGS_rdma_disable_bthread ? BTHREAD_ATTR_PTHREAD : BTHREAD_ATTR_NORMAL; attr.tag = tag; @@ -2021,11 +2027,23 @@ int RdmaEndpoint::PollingModeInitialize(bthread_tag_t tag, pollers[i].callback = callback; pollers[i].init_fn = init_fn; pollers[i].release_fn = release_fn; - auto rc = bthread_start_background(&pollers[i].tid, &attr, fn, args); +#if defined(UNIT_TEST) + const int rc = g_start_rdma_poller_for_test( +#else + const int rc = bthread_start_background( +#endif + &pollers[i].tid, &attr, fn, args.get()); if (rc != 0) { - LOG(ERROR) << "Fail to start rdma polling bthread"; + running.store(false, std::memory_order_relaxed); + for (size_t j = 0; j < i; ++j) { + bthread_join(pollers[j].tid, nullptr); + pollers[j].tid = INVALID_BTHREAD; + } + LOG(ERROR) << "Fail to start rdma polling bthread: " << berror(rc); + errno = rc; return -1; } + args.release(); } return 0; } @@ -2038,8 +2056,11 @@ void RdmaEndpoint::PollingModeRelease(bthread_tag_t tag) { auto& pollers = group.pollers; auto& running = group.running; running.store(false, std::memory_order_relaxed); - for (int i = 0; i < FLAGS_rdma_poller_num; ++i) { - bthread_join(pollers[i].tid, nullptr); + for (auto& poller : pollers) { + if (poller.tid != INVALID_BTHREAD) { + bthread_join(poller.tid, nullptr); + poller.tid = INVALID_BTHREAD; + } } } diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index 59bd9da55b..356f928190 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -23,6 +23,7 @@ #if BRPC_WITH_RDMA #include #include +#include #include #include #include @@ -32,6 +33,7 @@ #include "butil/sys_byteorder.h" #include "butil/time.h" #include "butil/files/temp_file.h" +#include "bthread/bthread.h" #include "brpc/acceptor.h" #include "brpc/channel.h" #include "brpc/controller.h" @@ -70,6 +72,8 @@ DECLARE_bool(rdma_trace_verbose); DECLARE_int32(rdma_memory_pool_max_regions); DECLARE_int32(rdma_client_handshake_version); DECLARE_bool(rdma_ece); +DECLARE_bool(rdma_poller_yield); +extern decltype(&bthread_start_background) g_start_rdma_poller_for_test; extern ibv_cq* (*IbvCreateCq)(ibv_context*, int, void*, ibv_comp_channel*, int); extern int (*IbvDestroyCq)(ibv_cq*); @@ -3569,6 +3573,125 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } +// Poller lifecycle tests do not need any RDMA queues or hardware. +static int g_poller_start_count = 0; +static int g_poller_fail_at = -1; + +static int StartRdmaPollerForTest(bthread_t* tid, const bthread_attr_t* attr, + void* (*fn)(void*), void* arg) { + if (g_poller_start_count++ == g_poller_fail_at) { + return EAGAIN; + } + return bthread_start_background(tid, attr, fn, arg); +} + +class RdmaPollingTest : public ::testing::TestWithParam { +protected: + void SetUp() override { + _saved_use_polling = rdma::FLAGS_rdma_use_polling; + _saved_poller_num = rdma::FLAGS_rdma_poller_num; + _saved_poller_yield = rdma::FLAGS_rdma_poller_yield; + _saved_disable_bthread = rdma::FLAGS_rdma_disable_bthread; + _saved_start_poller = rdma::g_start_rdma_poller_for_test; + rdma::FLAGS_rdma_use_polling = true; + rdma::FLAGS_rdma_poller_num = 3; + rdma::FLAGS_rdma_poller_yield = true; + _saved_groups.swap(rdma::RdmaEndpoint::_poller_groups); + rdma::RdmaEndpoint::_poller_groups = + std::vector(1); + rdma::g_start_rdma_poller_for_test = StartRdmaPollerForTest; + } + + void TearDown() override { + rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT); + rdma::g_start_rdma_poller_for_test = _saved_start_poller; + _saved_groups.swap(rdma::RdmaEndpoint::_poller_groups); + rdma::FLAGS_rdma_use_polling = _saved_use_polling; + rdma::FLAGS_rdma_poller_num = _saved_poller_num; + rdma::FLAGS_rdma_poller_yield = _saved_poller_yield; + rdma::FLAGS_rdma_disable_bthread = _saved_disable_bthread; + g_poller_fail_at = -1; + } + + void ExpectStopped() { + const auto& group = + rdma::RdmaEndpoint::_poller_groups[BTHREAD_TAG_DEFAULT]; + EXPECT_FALSE(group.running.load()); + for (const auto& poller : group.pollers) { + EXPECT_EQ(INVALID_BTHREAD, poller.tid); + } + } + + std::atomic _initialized{0}; + std::atomic _released{0}; + std::atomic _callbacks{0}; + +private: + bool _saved_use_polling; + int _saved_poller_num; + bool _saved_poller_yield; + bool _saved_disable_bthread; + decltype(&bthread_start_background) _saved_start_poller; + std::vector _saved_groups; +}; + +TEST_P(RdmaPollingTest, failed_creation_can_retry) { + // Exercise both the bthread and pthread-backed pollers. + for (bool disable_bthread : {false, true}) { + SCOPED_TRACE(disable_bthread); + rdma::FLAGS_rdma_disable_bthread = disable_bthread; + _initialized.store(0); + _released.store(0); + _callbacks.store(0); + g_poller_start_count = 0; + g_poller_fail_at = GetParam(); + auto callback = [this] { _callbacks.fetch_add(1); }; + auto init_fn = [this] { _initialized.fetch_add(1); }; + // Cleanup must preserve the creation error even if the callback + // changes errno. + auto release_fn = [this] { + _released.fetch_add(1); + errno = EINVAL; + }; + ASSERT_EQ(-1, rdma::RdmaEndpoint::PollingModeInitialize( + BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn)); + EXPECT_EQ(EAGAIN, errno); + EXPECT_EQ(GetParam() + 1, g_poller_start_count); + ExpectStopped(); + EXPECT_EQ(GetParam(), _initialized.load()); + EXPECT_EQ(GetParam(), _released.load()); + + // Release is safe even after a partially initialized group rolls back. + rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT); + ExpectStopped(); + EXPECT_EQ(GetParam(), _released.load()); + + g_poller_start_count = 0; + g_poller_fail_at = -1; + ASSERT_EQ(0, rdma::RdmaEndpoint::PollingModeInitialize( + BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn)); + ASSERT_TRUE(WaitUntil([this] { + return _initialized.load() == GetParam() + 3 && + _callbacks.load() > 0; + })); + EXPECT_EQ(3, g_poller_start_count); + EXPECT_TRUE(rdma::RdmaEndpoint::_poller_groups[BTHREAD_TAG_DEFAULT] + .running.load()); + EXPECT_EQ(0, rdma::RdmaEndpoint::PollingModeInitialize( + BTHREAD_TAG_DEFAULT, callback, init_fn, release_fn)); + EXPECT_EQ(3, g_poller_start_count); + + rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT); + ExpectStopped(); + EXPECT_EQ(GetParam() + 3, _released.load()); + rdma::RdmaEndpoint::PollingModeRelease(BTHREAD_TAG_DEFAULT); + EXPECT_EQ(GetParam() + 3, _released.load()); + } +} + +INSTANTIATE_TEST_SUITE_P(FailureAtEachPoller, RdmaPollingTest, + ::testing::Values(0, 1, 2)); + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) {