diff --git a/.github/workflows/ci-urma.yml b/.github/workflows/ci-urma.yml new file mode 100644 index 0000000000..174ddb9a2c --- /dev/null +++ b/.github/workflows/ci-urma.yml @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: URMA Mock + +on: + pull_request: + paths: + - 'src/**' + - 'test/**' + - 'cmake/**' + - 'CMakeLists.txt' + - 'example/urma_performance/**' + - '.github/workflows/ci-urma.yml' + push: + branches: [ master ] + paths: + - 'src/**' + - 'test/**' + - 'cmake/**' + - 'CMakeLists.txt' + - 'example/urma_performance/**' + - '.github/workflows/ci-urma.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + urma-mock: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v7.0.1 + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build libssl-dev libgflags-dev \ + libprotobuf-dev libprotoc-dev protobuf-compiler libleveldb-dev \ + libgtest-dev libgmock-dev libgoogle-perftools-dev + - name: Configure + run: | + cmake -S . -B build/urma-ci -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DWITH_RDMA=OFF -DWITH_URMA=ON \ + -DURMA_USE_MOCK=ON -DBUILD_UNIT_TESTS=ON -DDOWNLOAD_GTEST=OFF \ + -DBRPC_SYSTEM_GTEST_SOURCE_DIR=/usr/src/googletest \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + - name: Build URMA library and tests + run: | + targets=(brpc-static brpc_urma_unittest brpc_urma_recovery_unittest + brpc_urma_runtime_unittest brpc_urma_copy_unittest + brpc_urma_transport_unittest brpc_urma_remote_recovery_unittest + brpc_urma_mock_completion_unittest) + if test -f test/brpc_urma_qos_unittest.cpp; then + targets+=(brpc_urma_qos_unittest brpc_urma_qos_churn_unittest) + fi + cmake --build build/urma-ci --target "${targets[@]}" -j2 + - name: Run URMA regressions + run: | + ctest --test-dir build/urma-ci -R '^brpc_urma_.*unittest$' \ + --output-on-failure --timeout 120 --output-junit urma-results.xml + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: urma-test-results + path: | + build/urma-ci/urma-results.xml + build/urma-ci/Testing/Temporary/LastTest.log + if-no-files-found: ignore diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 62e10389a8..0c491c10cc 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -11,7 +11,7 @@ UrmaTransport 是使用 openEuler ## 技术背景 URMA 在 UMDK 支持的设备上提供 verbs 风格接口。当前实现创建可靠消息 -(`URMA_TM_RM`)Jetty,并使用 CTP 传输路径;通过 +(`URMA_TM_RM`)Jetty,并使用 CTP 或 RTP 传输路径;通过 `urma_post_jetty_send_wr` 提交发送 WR,通过 `urma_post_jfr_wr` 提交接收 WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获取。 @@ -20,22 +20,17 @@ WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获 ### CMake 编译 ```bash -# 带 URMA 支持编译 brpc -cmake -B build -DWITH_URMA=ON -make -C build -j$(nproc) - -# 编译 urma_performance 示例 -cd example/urma_performance -cmake -B build -make -C build -j$(nproc) +cmake -S . -B build -DWITH_URMA=ON +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/urma_performance -B build/urma-example \ + -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/urma-example -j8 ``` -`WITH_URMA=ON` 使用上游 UMDK 头文件进行编译。CMake 优先使用系统安装的 -SDK;找不到头文件时,会参照 Mooncake 的 mock 构建方式下载固定版本的 -UMDK commit,可通过 `DOWNLOAD_URMA_HEADERS=OFF` 禁止下载。找到 `liburma` 时使用 -真实硬件数据通路,否则链接 brpc 的 mock,使 URMA 代码和测试仍可在无硬件 -环境编译。`URMA_USE_MOCK` 默认为 `AUTO`;设为 `ON` 可强制使用 mock,设为 -`OFF` 则要求找到 `liburma` 并排除 mock。 +CMake 优先使用系统 UMDK 头文件,缺失时下载固定版本; +`DOWNLOAD_URMA_HEADERS=OFF` 关闭下载。`URMA_USE_MOCK=AUTO` 按 `liburma` 选择实现, +`ON` 使用 mock,`OFF` 使用硬件库。示例通过 `URMA_ROOT` 或 `URMA_INCLUDE_PATH` 指定 SDK 头文件。 ### Make 编译 @@ -60,7 +55,7 @@ sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \ # 编译带 URMA 支持的 brpc bazel build --define=BRPC_WITH_URMA=true //:brpc -# 仅展示关闭下载的开关;头文件仍须由工具链或本地 BUILD 目标声明 +# 使用本地声明的 SDK 头文件 bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc @@ -73,14 +68,9 @@ bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_URMA_USE_MOCK=false //:brpc ``` -Bazel 会自动获取固定 commit 的 UMDK 头文件,并通过 `@umdk//:urma_headers` -提供给 URMA 目标。设置 `BRPC_DOWNLOAD_URMA_HEADERS=false` 后,Bazel 不再依赖 -该外部仓库;实际构建仍需通过工具链或本地 BUILD 目标声明 -`urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。仅通过 -`--copt=-I/path` 指向工作区外目录,在默认 Bazel 沙箱下不会把未声明的头文件 -带入编译输入。设置 -`BRPC_URMA_USE_MOCK=true` 强制加入 mock;设置为 `false` 则排除 mock 并链接 -`liburma`。未指定时,URMA 构建默认使用 mock。 +Bazel 通过 `@umdk//:urma_headers` 使用固定版本的 UMDK 头文件。 +关闭下载时,由工具链或本地 BUILD 目标声明 `urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。 +`BRPC_URMA_USE_MOCK=true` 使用 mock,`false` 链接 `liburma`;默认使用 mock。 ## 使用 @@ -130,19 +120,15 @@ urma::UrmaEndpoint : public SocketUser (urma/urma_endpoint.{h,cpp}) 3. 双方在 TCP fd 上交换 `UrmaHello` 消息(v2 二进制 magic `URMA`, v3 protobuf magic `URM3`),携带本地 EID、jetty id、recv buffer 数量、 以及扁平化的 buffer 池 segment。 -4. 双方先调用 `urma_import_seg` **再**调用 `urma_import_jetty`,为远端 EID - 建立传输路径(TP)路由。跳过 `import_seg` 会导致首个 SEND 被硬件以 - `URMA_CR_RNR_RETRY_CNT_EXC_ERR` 拒绝。 +4. 双方依次调用 `urma_import_seg` 和 `urma_import_jetty`,建立远端路由。 5. 4 字节 ACK(`HELLO_ACK_URMA_OK = 0x1`)确认双方均要 URMA。 6. 成功后 TCP fd 仅保留用于 epoll 生命周期和回退,数据走 URMA。 ### 内存管理 -申请一大段 `mmap` 内存,用 `urma_register_seg` 一次性注册,再切成固定大小 -buffer(默认 8KB)。劫持 `butil::iobuf::blockmem_allocate` 使每个 IOBuf -block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 构建 -`urma_sge_t`,无需逐消息注册(与 RDMA `block_pool` 设计一致)。用户注册 -内存通过 `urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 支持。 +通过 `mmap` 分配内存,调用 `urma_register_seg` 注册后切为固定大小的缓冲区(默认 8KB)。 +IOBuf 使用注册池分配器,发送路径从 block refs 构建 `urma_sge_t`。 +用户内存通过 `RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 管理。 ## 配置 @@ -157,8 +143,10 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs | `--urma_rq_size` | 128 | 本地 JFR 深度 [16, 4096] | | `--urma_cqe_poll_once` | 32 | 每次 `urma_poll_jfc` 的上限 | | `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 | +| `--urma_copy_send` | false | 将主机发送数据复制合并到 64 字节对齐的注册缓冲区 | | `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | `--urma_device` | "" | URMA 设备名(空=首个) | +| `--urma_tp_type` | auto | RM 路径:auto 优先 CTP,其次 RTP;ctp 或 rtp 显式选择路径 | | `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) | | `--urma_bonding_mode` | 0 | bonding 模式:0=standalone,1=active-backup,2=balance | | `--urma_bonding_level` | 0 | bonding 层级:0=IODIE,1=port | @@ -172,6 +160,15 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 前配置 provider。默认 standalone+IODIE 配置与 UMDK 性能工具保持一致。 bonding 支持需要 provider 扩展头文件 `urma_ubagg.h`。 +### 传输路径选择 + +`--urma_tp_type=auto` 按设备能力优先选择 CTP,其次 RTP;`ctp`、`rtp` 显式指定路径。 +两端使用相同路径,进程初始化后保持固定。Jetty 取首个匹配优先级,支持组合能力位; +设备映射决定 SL,例如 priority 0 映射 SL 2。日志记录实际 `tp_type` 和 `priority`。 + +设备未提供能力表时,auto/ctp 使用 CTP priority 6,rtp 使用 RTP priority 0; +路径选择失败时初始化回落 TCP。 + ## 与 UBRing 协同 UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing @@ -185,11 +182,125 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing | 跨节点大包(>= 64KB) | `SOCKET_MODE_URMA` | | 传统 RoCE/IB 数据中心 | `SOCKET_MODE_URMA` 或 `SOCKET_MODE_RDMA` | -单连接内按包大小自动分流的方案(路线 C 方案 B)作为后续演进方向。 +## 运行环境 + +Linux、`baidu_std` 协议,以及支持目标设备的 UMDK provider 和 `liburma`。 + +## 运行期故障恢复 + +恢复策略按进程配置,作用于新连接: + +| 模式 | 行为 | +| --- | --- | +| LEGACY(默认) | 沿用原有握手和错误处理 | +| AUTO | 对端故障后冷却,期间使用 TCP,到期恢复 URMA 准入 | +| TCP_ONLY | 直接使用 TCP | + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +默认冷却 5 秒,故障来源包括重试耗尽、RNR、JFC 错误和 WR 提交失败。 +同一连接记录首个运行期故障;访问错误单独计数,窗口耗尽保持 EAGAIN。 +恢复策略维护进程级模式、epoch 和对端冷却缓存,模式切换过滤旧连接上报。 + +已建立 URMA 连接的客户端收到控制 TCP 连接关闭或读取错误时,上报传输故障, +AUTO 下冷却该对端。服务端收到正常短连接的 EOF 时直接关闭连接。 + +`GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态及协商数; +`/vars/urma_recovery_tcp_fallback_admissions` 按连接累计 TCP 准入。 + +### 功能验证 + +使用上文构建的 `urma_performance_server` 和 `urma_recovery_client`,两端指定相同设备路径。 +客户端示例: + +```bash +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ + --mode=auto --connection_type=short --seconds=15 --interval_ms=200 +``` + +健康 AUTO 检查 `failure=0`、URMA 协商增加、TCP 准入为 0; +TCP_ONLY 检查 `failure=0`、TCP 准入增加。运行中通过 `--mode_file` 切换模式。 +使用 `--connection_type=single` 或 `pooled` 验证连接复用,`--attachment_size=1048576` +验证 1 MiB 回显附件;客户端逐字节校验,汇总输出 `validation_failures`。 + +### 定向发送失败测试 + +启用 Linux 注入模块,在指定客户端模拟一次数据 SEND 提交失败(EIO): + +```bash +cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON +cmake --build build/urma-example --target urma_send_failure_injector -j8 +python3 example/urma_performance/run_recovery_injection.py \ + --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp +``` + +运行器等待 URMA RPC 成功后注入,校验指标 PID,保存 CSV、日志和三阶段快照。 +默认 v3 握手、运行 40 秒、冷却 10 秒;验收 `send_post=1` 和冷却期间 TCP 准入。 +`short` 检查一次预期 RPC 失败和冷却后 URMA 协商恢复;`--connection-type=single` 或 +`pooled` 检查重连后 RPC 持续成功,复用已建立的 TCP 连接。 +运行器支持 `--attachment-size` 回显校验和 `--polling` 轮询模式。 +构建 `urma_send_failure_injector_test` 后,通过示例目录的 CTest 执行工具自测。 + +主库 CTest 的 `brpc_urma_remote_recovery_*` 用例在服务端注入一次 SEND 失败, +覆盖 CTP/RTP、v2/v3、event/polling,检查客户端对端冷却、TCP 准入、 +URMA 恢复及 64 KiB 附件回显。正常短连接关闭也在同一测试中验证。 + +### 发送访问错误对照 + +`--urma_copy_send=true` 将每个数据 WR 合并到 64 字节对齐的注册缓冲区,并持有至完成, +增加一次 CPU 拷贝。两端保持设备、队列和消息大小一致,对比原发送路径与复制路径。 +初始化日志记录 `copy_send`,失败日志记录完成方向、槽位及注册段范围。 + +## 业务 SL 与优先级 + +同一进程可以同时运行对延迟敏感的 RPC 和批量传输。两类业务可以分别设置发送优先级, +未显式配置的业务继承进程默认值。`ChannelOptions.urma_qos` 控制请求发送方向, +`ServerOptions.urma_qos` 控制响应发送方向。 + +```cpp +brpc::ChannelOptions client_options; +client_options.socket_mode = brpc::SOCKET_MODE_URMA; +client_options.urma_qos.service_level = 2; + +brpc::ServerOptions server_options; +server_options.socket_mode = brpc::SOCKET_MODE_URMA; +server_options.urma_qos.service_level = 2; +``` + +`--urma_tp_type=auto` 按设备能力优先选择 CTP,其次 RTP;`ctp`、`rtp` 显式选择路径。 +两端使用相同路径,进程初始化后保持固定。设备未提供能力表时,auto/ctp 使用 priority 6, +rtp 使用 priority 0。 + +| 配置 | 默认 | 显式范围 | 含义 | +| --- | --- | --- | --- | +| `service_level` | -1 | 0–255 | 选择当前路径下首个映射到该 SL 的 priority | +| `priority` | -1 | 0–15 | 指定当前路径支持的 priority | + +两项均为 -1 时使用进程默认 priority。同时指定时,两项须匹配同一设备映射。 +每个设备在所选路径下最多提供 16 个 priority/SL 映射,实际可用项取决于设备能力。 +显式配置不受支持或发生冲突时,Channel 初始化或 Server 启动失败,日志会列出当前路径支持的映射。 +平台配置设备映射与调度策略,brpc 使用进程初始化时的映射快照。 + +客户端连接签名包含原始 QoS 设置。即使 RTP 的 priority 0 是 SL 2 的首个匹配项, +只填 `service_level=2` 和同时填 `priority=0` 的两个 Channel 也会分别建连。 +配置随服务发现、连接池、短连接和服务端 accept 传递,Jetty 重建时重新应用。 + +活跃的 URMA 连接在每端各占用一个 Jetty。匹配默认 priority 和队列大小的连接可以使用 +预建池中的 Jetty,其他连接按需创建 Jetty。v2/v3 握手使用所选路径。 + +### 功能验证 -## 限制 +用 `urma_admin show --dev "$URMA_DEVICE" --whole` 核对当前路径的 priority:SL。 +`example/urma_performance` 的两端均支持 `--urma_tp_type=rtp --urma_sl=2`, +也可通过 `--urma_priority` 指定匹配项。先以默认配置完成 RPC,再分别改变请求和响应 SL, +验证合法映射、不支持的 SL 和 SL/priority 冲突。对 v2/v3、single/pooled 及 polling 模式重复验证。 -- 仅支持 `baidu_std` 协议(与 RDMA 一致)。SSL、RTMP、NSHEAD、MONGO 在 - `ContextInitOrDie` 阶段拒绝。 -- 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 -- 当前实现面向 Linux。 +连接调试信息提供 `state`、`urma_effective_priority` 和 `urma_effective_sl`。 +核对 URMA 连接为 `ESTABLISHED`、映射符合设备配置、RPC 无错误。 +调度效果测试结合竞争流量、设备队列计数器、吞吐与 P99。 diff --git a/docs/en/urma.md b/docs/en/urma.md index a5b80f45b9..3252e9db1b 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -10,7 +10,7 @@ for large-packet / cross-node scenarios (Route C, "double backend"). ## Technical Background URMA exposes a verbs-style API over devices supported by UMDK. This -implementation creates a reliable-message (`URMA_TM_RM`) Jetty with a CTP +implementation creates a reliable-message (`URMA_TM_RM`) Jetty with a CTP or RTP transport path, posts send work requests with `urma_post_jetty_send_wr`, and posts receive work requests with `urma_post_jfr_wr`. Completions are consumed from a JFC either by busy polling or through a JFCE event fd. @@ -20,24 +20,18 @@ from a JFC either by busy polling or through a JFCE event fd. ### Build with CMake ```bash -# Build brpc with URMA support -cmake -B build -DWITH_URMA=ON -make -C build -j$(nproc) - -# Build the urma_performance example -cd example/urma_performance -cmake -B build -make -C build -j$(nproc) +cmake -S . -B build -DWITH_URMA=ON +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/urma_performance -B build/urma-example \ + -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/urma-example -j8 ``` -`WITH_URMA=ON` compiles against upstream UMDK headers. CMake prefers an -installed SDK and, following Mooncake's mock setup, downloads UMDK at a pinned -commit when the headers are unavailable. Set `DOWNLOAD_URMA_HEADERS=OFF` to -disable downloading. -When `liburma` is found it is linked for the hardware data path. Otherwise, -brpc uses its link-time mock so URMA code and tests can still be built without -hardware. `URMA_USE_MOCK` defaults to `AUTO`; set it to `ON` to force the mock, -or to `OFF` to require `liburma` and exclude the mock. +CMake uses installed UMDK headers or downloads a pinned version. +`DOWNLOAD_URMA_HEADERS=OFF` disables downloading. `URMA_USE_MOCK=AUTO` selects by +`liburma` availability; `ON` uses the mock and `OFF` uses the hardware library. +Set `URMA_ROOT` or `URMA_INCLUDE_PATH` to select SDK headers for the example. ### Build with Make @@ -63,7 +57,7 @@ requires `liburma` in the configured library paths. # Build brpc with URMA support bazel build --define=BRPC_WITH_URMA=true //:brpc -# This only shows the download switch; headers must still be declared by the toolchain or a local BUILD target +# Use locally declared SDK headers bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc @@ -76,15 +70,10 @@ bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_URMA_USE_MOCK=false //:brpc ``` -Bazel fetches the pinned UMDK commit and exposes its headers through -`@umdk//:urma_headers` by default. With -`BRPC_DOWNLOAD_URMA_HEADERS=false`, Bazel does not depend on that external -repository; the actual build must still declare `urma_api.h`, `urma_types.h`, -and `urma_ubagg.h` through the toolchain or a local BUILD target. Passing only -`--copt=-I/path` to a directory outside the workspace does not make undeclared -headers available inside Bazel's default sandbox. -`BRPC_URMA_USE_MOCK=true` forces the link-time mock, while `false` excludes it -and links `liburma`. If unspecified, URMA builds use the mock by default. +Bazel uses pinned UMDK headers through `@umdk//:urma_headers`. +With downloading disabled, declare `urma_api.h`, `urma_types.h` and `urma_ubagg.h` +through the toolchain or a local BUILD target. `BRPC_URMA_USE_MOCK=true` uses the +mock; `false` links `liburma`. The default is the mock. ## Usage @@ -137,23 +126,17 @@ Like RDMA / UBRing, the control plane is TCP and the data plane is URMA: 3. Both sides exchange a `UrmaHello` message (magic `URMA` for v2 binary, `URM3` for v3 protobuf) over the TCP fd. The message carries the local EID, jetty id, recv buffer count, and the flattened buffer-pool segment. -4. Each side calls `urma_import_seg` **before** `urma_import_jetty` to - establish transport-path (TP) routing for the remote EID. Skipping the - `import_seg` step causes the first SEND to be rejected by hardware with - `URMA_CR_RNR_RETRY_CNT_EXC_ERR`. +4. Each side calls `urma_import_seg`, then `urma_import_jetty`, to establish the remote route. 5. A 4-byte ACK (`HELLO_ACK_URMA_OK = 0x1`) confirms both sides want URMA. 6. On success, the TCP fd is kept only for the epoll lifecycle and fallback; payloads flow through URMA. ### Memory management -A single large region is `mmap`-ed and registered once with -`urma_register_seg`, then sliced into fixed-size buffers (default 8 KB). -`butil::iobuf::blockmem_allocate` is hijacked so every IOBuf block is backed -by the registered segment, allowing the send path to build `urma_sge_t` -directly from IOBuf block refs without per-message registration (mirroring the -RDMA `block_pool` design). User-registered memory is supported via -`urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma`. +A region allocated with `mmap` is registered with `urma_register_seg` and split +into fixed-size buffers (default 8 KB). IOBuf uses the registered-pool allocator; +the send path builds `urma_sge_t` from block refs. Manage user memory with +`RegisterMemoryForUrma` / `DeregisterMemoryForUrma`. ## Configuration @@ -168,8 +151,10 @@ All flags use the `urma_` prefix (mirroring RDMA's `rdma_` prefix): | `--urma_rq_size` | 128 | Local JFR depth [16, 4096] | | `--urma_cqe_poll_once` | 32 | Max CQEs per `urma_poll_jfc` | | `--urma_recv_zerocopy` | true | Zero-copy receives above `--urma_zerocopy_min_size` | +| `--urma_copy_send` | false | Copy and coalesce host payloads into a 64-byte aligned registered send buffer | | `--urma_zerocopy_min_size` | 512 | Receives smaller than this are copied | | `--urma_device` | "" | URMA device name (empty = first) | +| `--urma_tp_type` | auto | RM path: auto prefers CTP, then RTP; ctp or rtp selects a path explicitly | | `--urma_max_sge` | 0 | Max SGEs per WR (0 = device max) | | `--urma_bonding_mode` | 0 | Bonding mode: 0=standalone, 1=active-backup, 2=balance | | `--urma_bonding_level` | 0 | Bonding level: 0=IODIE, 1=port | @@ -184,6 +169,17 @@ immediately after context creation and before creating segments or queues. The default standalone+IODIE combination matches the UMDK performance tool. Bonding support requires the provider extension header `urma_ubagg.h`. +### Transport path selection + +`--urma_tp_type=auto` prefers CTP, then RTP, from device capabilities; `ctp` and +`rtp` select a path explicitly. Both peers use the same path, fixed at process +initialization. Jettys use the first matching priority, including combined +capability bits. The device maps priority to SL, such as priority 0 to SL 2. +Logs report the selected `tp_type` and `priority`. + +Providers without a capability table use CTP priority 6 for auto/ctp or RTP +priority 0 for rtp. Path selection failure uses initialization's TCP fallback. + ## Coexistence with UBRing UrmaTransport is the recommended transport for **large packets and @@ -198,12 +194,147 @@ syscalls). For a mixed workload, select the transport per service: | Cross-node large packets (>= 64 KB) | `SOCKET_MODE_URMA` | | Traditional RoCE / IB datacenter | `SOCKET_MODE_URMA` or `SOCKET_MODE_RDMA` | -A single-connection hybrid that auto-routes by packet size (Route C, scheme B) -is tracked as a future enhancement. +## Runtime environment + +Linux, the `baidu_std` protocol, and a UMDK provider with `liburma` for the target device. + +## Runtime recovery + +Configure recovery per process for new connections: + +| Mode | Behavior | +| --- | --- | +| LEGACY (default) | Existing handshake and error handling | +| AUTO | Cool a failed peer, use TCP during cooldown, then resume URMA admission | +| TCP_ONLY | Select TCP | + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +The default cooldown is five seconds. Sources include retry exhaustion, RNR, +JFC errors and WR submission failures. Each connection records its first runtime +failure; access errors have a separate counter, and exhausted windows return EAGAIN. +The recovery policy maintains process-level modes, epochs and peer cooldowns. +Mode changes filter reports from older connections. + +A client reports a transport failure when an established URMA connection's +control TCP socket closes or fails to read, cooling the peer in AUTO mode. +Server-side EOF from a normal short-connection client simply closes the socket. + +`GetUrmaRecoveryStatus()` and `/vars/urma_recovery_status` expose failure reasons, +cooldown state and negotiation counts. `/vars/urma_recovery_tcp_fallback_admissions` +counts TCP admissions per connection. + +### Functional checks + +Use the `urma_performance_server` and `urma_recovery_client` built above with the +same path on both peers. Client example: -## Limitations +```bash +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ + --mode=auto --connection_type=short --seconds=15 --interval_ms=200 +``` + +Healthy AUTO requires `failure=0`, increasing URMA negotiations and zero TCP +admissions. TCP_ONLY requires `failure=0` and increasing TCP admissions. +Use `--mode_file` to switch modes while running. +Use `--connection_type=single` or `pooled` for connection reuse and +`--attachment_size=1048576` for a 1 MiB echo attachment. The client checks every +byte and reports `validation_failures` in the summary. + +### Targeted send failure test + +Enable the Linux injector to reject one data SEND with EIO in the selected client: + +```bash +cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON +cmake --build build/urma-example --target urma_send_failure_injector -j8 +python3 example/urma_performance/run_recovery_injection.py \ + --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp +``` + +The runner waits for successful URMA RPCs, checks the metric listener PID, and +saves CSV, logs and three phase snapshots. Defaults are v3, a 40-second run and +10-second cooldown. Checks require `send_post=1` and TCP admission during cooldown. +`short` requires one expected RPC failure and resumed URMA negotiations; +`--connection-type=single` or `pooled` checks sustained RPC recovery after +reconnecting and reusing TCP. The runner supports `--attachment-size` for echo +validation and `--polling` for polling mode. +Build `urma_send_failure_injector_test` and run CTest in the example build +directory for the tool self-tests. + +The main library's `brpc_urma_remote_recovery_*` CTest cases inject one server +SEND failure across CTP/RTP, v2/v3 and event/polling modes. They check client +peer cooldown, TCP admission, URMA recovery and 64 KiB attachment echoes, +including normal short-connection closes before the injection. + +### Comparing send access failures + +`--urma_copy_send=true` coalesces each data WR into a 64-byte aligned registered +buffer held until completion, adding one CPU copy. Keep device, queue and payload +settings consistent when comparing send paths. Startup logs report `copy_send`; +failure logs report direction, send slot and registered segment range. + +## Business SL and priority + +A process can serve latency-sensitive RPCs alongside bulk transfers. Configure +their outgoing priorities independently and leave other traffic on the process +default. Set `ChannelOptions.urma_qos` for outgoing requests and +`ServerOptions.urma_qos` for outgoing responses. + +```cpp +brpc::ChannelOptions client_options; +client_options.socket_mode = brpc::SOCKET_MODE_URMA; +client_options.urma_qos.service_level = 2; + +brpc::ServerOptions server_options; +server_options.socket_mode = brpc::SOCKET_MODE_URMA; +server_options.urma_qos.service_level = 2; +``` -- `baidu_std` protocol only (same as RDMA). SSL, RTMP, NSHEAD, MONGO are - rejected at `ContextInitOrDie`. -- A hardware data path requires a supported UMDK provider and `liburma`. -- The current implementation targets Linux. +`--urma_tp_type=auto` prefers CTP, then RTP, using device capabilities. +Use `ctp` or `rtp` to select a path explicitly. Both peers use the same path, +fixed at process initialization. Providers without a capability table use +priority 6 for auto/CTP and priority 0 for RTP. + +| Option | Default | Explicit range | Meaning | +| --- | --- | --- | --- | +| `service_level` | -1 | 0–255 | Select the first priority mapping to this SL on the selected path | +| `priority` | -1 | 0–15 | Select a priority supported by the selected path | + +Both fields at -1 select the process default. When both are explicit, they must +match the same device mapping. Each device exposes up to 16 priority/SL mappings +on the selected path, depending on its capabilities. Channel initialization and +Server startup reject unsupported or conflicting settings and log the available +mappings. The platform configures mappings and scheduling. brpc uses the +initialization snapshot. + +The client connection signature includes the configured QoS values. If RTP +priority 0 is the first mapping for SL 2, a channel that sets only +`service_level=2` uses a separate connection from one that also sets +`priority=0`. Naming services, pooled and short connections, accepted sockets +and rebuilt Jettys inherit the configuration. + +Each active URMA connection uses one Jetty at each end. Connections with the +default priority and a compatible queue size can use a Jetty from the prepared +pool. Other connections allocate Jettys on demand. Both v2 and v3 use the +selected path. + +### Functional validation + +Inspect the selected path's priority:SL map with +`urma_admin show --dev "$URMA_DEVICE" --whole`. +Both examples in `example/urma_performance` accept `--urma_tp_type=rtp --urma_sl=2` +and an optional matching `--urma_priority`. Establish RPCs with defaults, then +change request and response SLs separately. Check supported mappings, unsupported +SLs and conflicting pairs across v2/v3, single/pooled and polling modes. + +Connection diagnostics expose `state`, `urma_effective_priority` and +`urma_effective_sl`. Check that URMA is `ESTABLISHED`, the mapping matches the +device and RPCs succeed. Evaluate scheduling with competing traffic, device queue +counters, throughput and P99. diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 4dbc989a45..3b92b83720 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -21,6 +21,7 @@ project(urma_performance C CXX) include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake) option(LINK_SO "Whether examples are linked dynamically" OFF) +option(BUILD_URMA_FAULT_INJECTOR "Build the process-local SEND failure test module" OFF) brpc_example_find_common_deps(DYNAMIC_LIB) @@ -52,3 +53,39 @@ brpc_example_configure_target(urma_performance_server) target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +add_executable(urma_recovery_client + recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) +brpc_example_configure_target(urma_recovery_client) +target_link_libraries(urma_recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +if(BUILD_URMA_FAULT_INJECTOR) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The URMA fault injector uses Linux LD_PRELOAD") + endif() + add_library(urma_send_failure_injector MODULE send_failure_injector.c) + set_target_properties(urma_send_failure_injector PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON) + target_include_directories(urma_send_failure_injector PRIVATE ${URMA_INCLUDE_PATH}) + target_link_libraries(urma_send_failure_injector PRIVATE ${CMAKE_DL_LIBS} Threads::Threads) + include(CTest) + if(BUILD_TESTING) + add_library(urma_injector_test_provider SHARED test_send_failure_injector.c) + target_compile_definitions(urma_injector_test_provider PRIVATE URMA_INJECTOR_TEST_PROVIDER) + target_include_directories(urma_injector_test_provider PRIVATE ${URMA_INCLUDE_PATH}) + add_executable(urma_send_failure_injector_test test_send_failure_injector.c) + target_include_directories(urma_send_failure_injector_test PRIVATE ${URMA_INCLUDE_PATH}) + target_link_libraries(urma_send_failure_injector_test PRIVATE urma_injector_test_provider) + add_dependencies(urma_send_failure_injector_test urma_send_failure_injector) + add_test(NAME urma_send_failure_injector_test + COMMAND ${CMAKE_COMMAND} -E env + "LD_PRELOAD=$" + $) + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + add_test(NAME urma_recovery_injection_runner_test + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_run_recovery_injection.py) + endif() + endif() +endif() diff --git a/example/urma_performance/client.cpp b/example/urma_performance/client.cpp index 9449dbe960..8f7e29570a 100644 --- a/example/urma_performance/client.cpp +++ b/example/urma_performance/client.cpp @@ -44,6 +44,9 @@ DEFINE_int32(attachment_size, -1, "Attachment size is used (in Bytes)"); DEFINE_int32(rpc_timeout_ms, 5000, "Timeout for each RPC in milliseconds"); DEFINE_bool(echo_attachment, false, "Select whether attachment should be echo"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); +DEFINE_int32(urma_priority, -1, + "Outbound URMA priority: -1 automatic, or [0,15]"); +DEFINE_int32(urma_sl, -1, "Outbound URMA SL: -1 unspecified, or [0,255]"); bvar::LatencyRecorder g_latency("client"); bvar::Adder g_error_count("client_error_count"); @@ -88,6 +91,8 @@ static void* worker(void* arg) { int main(int argc, char* argv[]) { gflags::ParseCommandLineFlags(&argc, &argv, true); brpc::ChannelOptions options; + options.urma_qos.priority = FLAGS_urma_priority; + options.urma_qos.service_level = FLAGS_urma_sl; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; options.connect_timeout_ms = FLAGS_rpc_timeout_ms; @@ -110,7 +115,9 @@ int main(int argc, char* argv[]) { stub.Test(&warmup_cntl, &warmup_req, &warmup_resp, nullptr); if (warmup_cntl.Failed()) { LOG(ERROR) << "Warm-up RPC failed after timeout_ms=" - << FLAGS_rpc_timeout_ms << ": " + << FLAGS_rpc_timeout_ms << " use_urma=" << FLAGS_use_urma + << " urma_priority=" << FLAGS_urma_priority + << " urma_sl=" << FLAGS_urma_sl << ": " << warmup_cntl.ErrorText(); return -1; } @@ -131,7 +138,8 @@ int main(int argc, char* argv[]) { } LOG(INFO) << "URMA performance client started (server=" << FLAGS_server << ", use_urma=" << FLAGS_use_urma - << ", threads=" << thread_num + << ", urma_priority=" << FLAGS_urma_priority + << ", urma_sl=" << FLAGS_urma_sl << ", threads=" << thread_num << ", rpc_timeout_ms=" << FLAGS_rpc_timeout_ms << ")"; while (!brpc::IsAskedToQuit()) { sleep(1); diff --git a/example/urma_performance/recovery_client.cpp b/example/urma_performance/recovery_client.cpp new file mode 100644 index 0000000000..a1a53fe676 --- /dev/null +++ b/example/urma_performance/recovery_client.cpp @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/time.h" +#include "test.pb.h" + +DEFINE_string(server, "127.0.0.1:8003", "Performance test server address"); +DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); +DEFINE_string(mode_file, "", + "Read a recovery mode from this file while running"); +DEFINE_string(connection_type, "short", + "short exercises new admission on each RPC"); +DEFINE_string(transport, "urma", "urma or tcp"); +DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); +DEFINE_int32(interval_ms, 100, "Pause between RPCs"); +DEFINE_int32(timeout_ms, 1000, "RPC deadline"); +DEFINE_int32(cooldown_ms, 5000, "Path cooldown"); +DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); +DEFINE_int32(attachment_size, 0, + "Echo attachment bytes and verify the returned content"); + +static bool SetMode(const std::string& value) { + using brpc::urma::UrmaRecoveryMode; + UrmaRecoveryMode mode; + if (value == "legacy") { + mode = UrmaRecoveryMode::LEGACY; + } else if (value == "auto") { + mode = UrmaRecoveryMode::AUTO; + } else if (value == "tcp_only") { + mode = UrmaRecoveryMode::TCP_ONLY; + } else { + return false; + } + return brpc::urma::SetUrmaRecoveryMode(mode); +} + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_transport != "urma" && FLAGS_transport != "tcp") { + std::cerr << "Invalid transport\n"; + return 1; + } + const bool cooldown_ok = + brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms); + if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || + FLAGS_attachment_size < 0 || !cooldown_ok || !SetMode(FLAGS_mode)) { + std::cerr + << "Invalid duration, interval, timeout, cooldown, mode or attachment size\n"; + return 1; + } + brpc::Channel channel; + brpc::ChannelOptions options; + options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA + : brpc::SOCKET_MODE_TCP; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = FLAGS_timeout_ms; + options.max_retry = 0; + if (channel.Init(FLAGS_server.c_str(), &options) != 0) { + return 1; + } + if (FLAGS_dummy_port && brpc::StartDummyServerAt(FLAGS_dummy_port) != 0) { + return 1; + } + std::string mode = FLAGS_mode; + uint64_t succeeded = 0; + uint64_t failed = 0; + uint64_t validation_failed = 0; + std::string payload(static_cast(FLAGS_attachment_size), '\0'); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i % 251); + } + const int64_t start = butil::monotonic_time_us(); + int64_t next_report = start; + std::cout + << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," + "cooled_peers\n"; + while (!brpc::IsAskedToQuit() && + butil::monotonic_time_us() - start < + static_cast(FLAGS_seconds) * 1000000) { + if (!FLAGS_mode_file.empty()) { + std::ifstream input(FLAGS_mode_file); + std::string value; + if (input >> value && value != mode) { + if (SetMode(value)) { + mode = value; + } else { + std::cerr << "Ignoring invalid recovery mode: " << value + << '\n'; + } + } + } + test::PerfTestRequest request; + request.set_echo_attachment(!payload.empty()); + test::PerfTestResponse response; + brpc::Controller cntl; + const uint64_t sequence = succeeded + failed; + for (size_t i = 0; i < payload.size() && i < sizeof(sequence); ++i) { + payload[i] = static_cast(sequence >> (i * 8)); + } + cntl.request_attachment().append(payload); + test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, + nullptr); + if (cntl.Failed()) { + ++failed; + std::cerr << "RPC failure: " << cntl.ErrorText() << '\n'; + } else if (!payload.empty() && + cntl.response_attachment().to_string() != payload) { + ++failed; + ++validation_failed; + std::cerr << "Attachment mismatch: expected=" << payload.size() + << " actual=" << cntl.response_attachment().size() + << '\n'; + } else { + ++succeeded; + } + const int64_t now = butil::monotonic_time_us(); + if (now >= next_report) { + const auto s = brpc::urma::GetUrmaRecoveryStatus(); + std::cout << (now - start) / 1000000 << ',' << mode << ',' + << succeeded << ',' << failed << ',' + << s.negotiated_connections << ',' + << s.tcp_only_admissions + s.peer_cooldown_admissions + << ',' << s.cooled_peers << std::endl; + next_report = now + 1000000; + } + usleep(static_cast(FLAGS_interval_ms) * 1000); + } + std::cout << "completed success=" << succeeded << " failure=" << failed; + if (!payload.empty()) { + std::cout << " validation_failures=" << validation_failed; + } + std::cout << std::endl; + return succeeded ? 0 : 2; +} diff --git a/example/urma_performance/run_recovery_injection.py b/example/urma_performance/run_recovery_injection.py new file mode 100644 index 0000000000..5d021b01cc --- /dev/null +++ b/example/urma_performance/run_recovery_injection.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check one SEND failure, peer cooldown, TCP admission and URMA recovery.""" + +import argparse +import csv +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import tempfile +import time +import urllib.request + + +def decode_bvar(text): + match = re.match(r"^\w+\s*:\s*(.*)$", text.strip(), re.DOTALL) + value = match.group(1) if match else text.strip() + try: + result = json.loads(value) + return json.loads(result) if isinstance(result, str) else result + except json.JSONDecodeError: + # Text /vars may quote the JSON object without escaping its quotes. + return json.loads(value[value.index("{"):value.rindex("}") + 1]) + + +def read_samples(path): + with path.open() as stream: + return [list(map(int, row[:1] + row[2:])) for row in csv.reader(stream) + if len(row) == 7 and row[0].isdigit()] + + +def wait_for(proc, predicate, timeout, description): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + if proc.poll() is not None: + raise RuntimeError("Client exited while waiting for " + description) + time.sleep(0.1) + raise RuntimeError("Timed out waiting for " + description) + + +def validate_phases(baseline, cooldown, recovered, require_urma_reconnect=True): + if any(s["mode"] != "auto" for s in (baseline, cooldown, recovered)): + raise RuntimeError("Expected AUTO throughout the run") + if len({s["epoch"] for s in (baseline, cooldown, recovered)}) != 1: + raise RuntimeError("Recovery epoch changed during injection") + if (baseline["negotiated_connections"] <= 0 or baseline["cooled_peers"] != 0 + or baseline["failures"]["send_post"] != 0): + raise RuntimeError("Baseline must use URMA with no injected failure") + if (cooldown["failures"]["send_post"] != 1 or cooldown["cooled_peers"] != 1 + or cooldown["peer_cooldown_admissions"] <= 0): + raise RuntimeError("Expected one SEND failure and TCP peer-cooldown admission") + negotiated = recovered["negotiated_connections"] + previous = cooldown["negotiated_connections"] + if (recovered["failures"]["send_post"] != 1 or recovered["cooled_peers"] != 0 + or negotiated < previous or (require_urma_reconnect and negotiated == previous) + or recovered["peer_cooldown_admissions"] < cooldown["peer_cooldown_admissions"]): + raise RuntimeError("Expected URMA negotiations to resume after cooldown") + + +def run(args): + client = Path(args.client).resolve() + injector = Path(args.injector).resolve() + if not client.is_file() or not injector.is_file(): + raise RuntimeError("Build urma_recovery_client and urma_send_failure_injector first") + output = Path(tempfile.mkdtemp(prefix="urma-injection-", dir=".")).resolve() + marker = output / "inject.once" + csv_path = output / "client.csv" + log_path = output / "client.log" + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + # All metric requests go directly to this client's loopback listener. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def read_var(name): + request = urllib.request.Request( + "http://127.0.0.1:{}/vars/{}?console=1".format(port, name), + headers={"Accept": "text/plain"}) + with opener.open(request, timeout=2) as response: + text = response.read().decode() + (output / ("latest-" + name + ".txt")).write_text(text) + return decode_bvar(text) + + def record_phase(name, status): + (output / (name + ".json")).write_text(json.dumps(status, indent=2)) + print("{}: send_post={} cooled_peers={} tcp={} negotiated={}".format( + name, status["failures"]["send_post"], status["cooled_peers"], + status["peer_cooldown_admissions"], status["negotiated_connections"]), flush=True) + return status + + command = [str(client), "--transport=urma", "--mode=auto", + "--server=" + args.server, "--urma_device=" + args.device, + "--urma_tp_type=" + args.tp_type, + "--urma_client_handshake_version=" + str(args.handshake_version), + "--urma_buffer_count=4096", "--connection_type=" + args.connection_type, + "--attachment_size=" + str(args.attachment_size), + "--urma_use_polling=" + str(args.polling).lower(), + "--interval_ms=200", "--timeout_ms=3000", + "--seconds=" + str(args.seconds), + "--cooldown_ms=" + str(args.cooldown_ms), "--dummy_port=" + str(port)] + env = dict(os.environ, LD_PRELOAD=str(injector), + BRPC_URMA_FAIL_ONCE_FILE=str(marker)) + proc = None + print("Artifacts: " + str(output), flush=True) + try: + with csv_path.open("w") as stdout, log_path.open("w") as stderr: + proc = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env) + print("Client PID={}; waiting for successful URMA RPCs".format(proc.pid), flush=True) + + def ready(): + rows = read_samples(csv_path) + if rows and rows[-1][2] != 0: + raise RuntimeError("RPC failed before injection") + return rows and rows[-1][1] > 0 and rows[-1][3] > 0 + + wait_for(proc, ready, args.startup_timeout, "successful URMA baseline") + if read_var("pid") != proc.pid: + raise RuntimeError("Metric listener PID does not match the test client") + time.sleep(args.baseline_seconds) + phases = {"baseline": record_phase("baseline", read_var("urma_recovery_status"))} + baseline = phases["baseline"] + if (baseline["failures"]["send_post"] or baseline["cooled_peers"] + or baseline["negotiated_connections"] <= 0): + raise RuntimeError("Unexpected baseline metrics") + marker.touch() + + def cooling(): + status = read_var("urma_recovery_status") + return status if (status["failures"]["send_post"] == 1 + and status["cooled_peers"] == 1 + and status["peer_cooldown_admissions"] > 0) else None + + phases["cooldown"] = record_phase("cooldown", wait_for( + proc, cooling, args.cooldown_ms / 1000, "peer cooldown and TCP admission")) + require_urma_reconnect = args.connection_type == "short" + recovery_success = None + + def recovering(): + nonlocal recovery_success + status = read_var("urma_recovery_status") + if status["cooled_peers"] != 0: + return None + if require_urma_reconnect: + return status if (status["negotiated_connections"] > + phases["cooldown"]["negotiated_connections"]) else None + rows = read_samples(csv_path) + if recovery_success is None: + recovery_success = rows[-1][1] + return status if rows[-1][1] > recovery_success else None + + phases["recovered"] = record_phase("recovered", wait_for( + proc, recovering, args.cooldown_ms / 1000 + 5, "URMA recovery")) + validate_phases(**phases, require_urma_reconnect=require_urma_reconnect) + failures_at_recovery = read_samples(csv_path)[-1][2] + (output / "snapshots.json").write_text(json.dumps( + {"pid": proc.pid, "port": port, "phases": phases}, indent=2)) + rc = proc.wait(timeout=args.seconds + 10) + summary = re.search(r"completed success=(\d+) failure=(\d+)", csv_path.read_text()) + expected_failures = 1 if require_urma_reconnect else failures_at_recovery + if (rc != 0 or not summary or int(summary[1]) == 0 + or expected_failures < 1 or int(summary[2]) != expected_failures): + raise RuntimeError("Expected RPC recovery with no further failures afterward") + if args.attachment_size: + validation = re.search(r"validation_failures=(\d+)", csv_path.read_text()) + if not validation or int(validation[1]) != 0: + raise RuntimeError("Attachment content validation failed") + if log_path.read_text().count("injected one data SEND post failure") != 1: + raise RuntimeError("Expected exactly one injection log") + last = read_samples(csv_path)[-1] + if last[5] != 0 or last[4] != phases["recovered"]["peer_cooldown_admissions"]: + raise RuntimeError("TCP cooldown admissions must stop after URMA recovery") + print(csv_path.read_text().splitlines()[-1]) + recovered_transport = "URMA" if require_urma_reconnect else "RPC" + print("PASS: one SEND failure, TCP cooldown, and " + recovered_transport + " recovery") + finally: + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True) + parser.add_argument("--device", required=True) + parser.add_argument("--client", default="build/urma-example/urma_recovery_client") + parser.add_argument("--injector", default="build/urma-example/liburma_send_failure_injector.so") + parser.add_argument("--tp-type", choices=("rtp", "ctp", "auto"), default="rtp") + parser.add_argument("--handshake-version", type=int, choices=(2, 3), default=3) + parser.add_argument("--connection-type", choices=("short", "single", "pooled"), default="short") + parser.add_argument("--attachment-size", type=int, default=0) + parser.add_argument("--polling", action="store_true") + parser.add_argument("--seconds", type=int, default=40) + parser.add_argument("--cooldown-ms", type=int, default=10000) + parser.add_argument("--baseline-seconds", type=float, default=5) + parser.add_argument("--startup-timeout", type=float, default=120) + args = parser.parse_args() + if (args.cooldown_ms <= 0 or args.attachment_size < 0 + or args.baseline_seconds < 0 or args.startup_timeout <= 0 + or args.seconds < args.baseline_seconds + args.cooldown_ms / 1000 + 5): + parser.error("Duration must cover baseline, cooldown and at least five seconds of recovery") + try: + run(args) + except (OSError, ValueError, KeyError, TypeError, RuntimeError, subprocess.TimeoutExpired) as error: + parser.exit(1, "FAIL: {}\n".format(error)) + + +if __name__ == "__main__": + main() diff --git a/example/urma_performance/send_failure_injector.c b/example/urma_performance/send_failure_injector.c new file mode 100644 index 0000000000..60b028a2af --- /dev/null +++ b/example/urma_performance/send_failure_injector.c @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "urma_api.h" + +typedef urma_status_t (*PostSend)(urma_jetty_t*, urma_jfs_wr_t*, + urma_jfs_wr_t**); +static PostSend g_post_send; +static pthread_once_t g_resolve_once = PTHREAD_ONCE_INIT; +static atomic_bool g_injected = ATOMIC_VAR_INIT(0); + +static void ResolvePostSend(void) { + g_post_send = (PostSend)dlsym(RTLD_NEXT, "urma_post_jetty_send_wr"); + if (!g_post_send) { + fprintf(stderr, "URMA injector: cannot resolve real post_send: %s\n", + dlerror()); + } +} + +// LD_PRELOAD this module into the recovery client. Creating the marker named +// by BRPC_URMA_FAIL_ONCE_FILE rejects one data SEND before it reaches liburma. +// Credit ACKs and all subsequent sends are forwarded to the provider. +urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, + urma_jfs_wr_t** bad_wr) { + pthread_once(&g_resolve_once, ResolvePostSend); + if (!g_post_send) { + if (bad_wr) { + *bad_wr = wr; + } + errno = ENOSYS; + return ENOSYS; + } + const char* marker = getenv("BRPC_URMA_FAIL_ONCE_FILE"); + if (wr && wr->opcode == URMA_OPC_SEND && marker && marker[0] && + access(marker, F_OK) == 0 && !atomic_exchange(&g_injected, 1)) { + if (unlink(marker) != 0) { + fprintf(stderr, "URMA injector: cannot remove marker: %s\n", + marker); + } + if (bad_wr) { + *bad_wr = wr; + } + fprintf( + stderr, + "URMA injector: injected one data SEND post failure, errno=%d\n", + EIO); + errno = EIO; + return EIO; + } + return g_post_send(jetty, wr, bad_wr); +} diff --git a/example/urma_performance/server.cpp b/example/urma_performance/server.cpp index baaff323b6..e5f8649487 100644 --- a/example/urma_performance/server.cpp +++ b/example/urma_performance/server.cpp @@ -30,6 +30,9 @@ DEFINE_int32(port, 8003, "TCP Port of this server"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); +DEFINE_int32(urma_priority, -1, + "Outbound URMA priority: -1 automatic, or [0,15]"); +DEFINE_int32(urma_sl, -1, "Outbound URMA SL: -1 unspecified, or [0,255]"); butil::atomic g_last_time(0); @@ -73,6 +76,8 @@ int main(int argc, char* argv[]) { } brpc::ServerOptions options; + options.urma_qos.priority = FLAGS_urma_priority; + options.urma_qos.service_level = FLAGS_urma_sl; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; if (server.Start(FLAGS_port, &options) != 0) { @@ -80,7 +85,9 @@ int main(int argc, char* argv[]) { return -1; } LOG(INFO) << "URMA performance server started on port " << FLAGS_port - << " (use_urma=" << FLAGS_use_urma << ")"; + << " (use_urma=" << FLAGS_use_urma + << ", urma_priority=" << FLAGS_urma_priority + << ", urma_sl=" << FLAGS_urma_sl << ")"; server.RunUntilAskedToQuit(); return 0; } diff --git a/example/urma_performance/test_run_recovery_injection.py b/example/urma_performance/test_run_recovery_injection.py new file mode 100644 index 0000000000..472229b930 --- /dev/null +++ b/example/urma_performance/test_run_recovery_injection.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +sys.dont_write_bytecode = True +import run_recovery_injection as runner + + +class RecoveryInjectionTest(unittest.TestCase): + def test_decode_text_and_json_bvar_formats(self): + value = {"mode": "auto", "failures": {"send_post": 1}} + raw = json.dumps(value) + for body in (raw, 'urma_recovery_status : "' + raw + '"', + "urma_recovery_status : " + json.dumps(raw)): + self.assertEqual(value, runner.decode_bvar(body)) + self.assertEqual(123, runner.decode_bvar("pid : 123\n")) + + def test_samples_skip_headers_summary_and_partial_rows(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "client.csv" + path.write_text("seconds,mode,success,failure\n" + "0,auto,1,0,1,0,0\n1,auto," + "\ncompleted success=1 failure=0\n") + self.assertEqual([[0, 1, 0, 1, 0, 0]], runner.read_samples(path)) + + def phases(self): + baseline = {"mode": "auto", "epoch": 2, "negotiated_connections": 10, + "cooled_peers": 0, "peer_cooldown_admissions": 0, + "failures": {"send_post": 0}} + cooldown = copy.deepcopy(baseline) + cooldown.update(cooled_peers=1, peer_cooldown_admissions=3) + cooldown["failures"]["send_post"] = 1 + recovered = copy.deepcopy(cooldown) + recovered.update(cooled_peers=0, negotiated_connections=12) + return baseline, cooldown, recovered + + def test_validate_complete_recovery(self): + runner.validate_phases(*self.phases()) + + def test_persistent_connections_can_reuse_tcp_after_cooldown(self): + baseline, cooldown, recovered = self.phases() + recovered["negotiated_connections"] = cooldown["negotiated_connections"] + runner.validate_phases(baseline, cooldown, recovered, require_urma_reconnect=False) + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + + def test_reject_stale_zero_failure_snapshot(self): + baseline, cooldown, recovered = self.phases() + cooldown["failures"]["send_post"] = 0 + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + + def test_reject_no_recovery_and_epoch_changes(self): + for field, value in (("negotiated_connections", 10), ("epoch", 3)): + baseline, cooldown, recovered = self.phases() + recovered[field] = value + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + + def test_wait_for_readiness_instead_of_fixed_startup_delay(self): + proc = mock.Mock() + proc.poll.return_value = None + ready = mock.Mock(side_effect=[False, False, "ready"]) + with mock.patch.object(runner.time, "sleep"): + self.assertEqual("ready", runner.wait_for(proc, ready, 1, "baseline")) + self.assertEqual(3, ready.call_count) + + def test_exit_before_readiness_is_a_failure(self): + proc = mock.Mock() + proc.poll.return_value = 1 + with self.assertRaises(RuntimeError): + runner.wait_for(proc, lambda: False, 1, "baseline") + + +if __name__ == "__main__": + unittest.main() diff --git a/example/urma_performance/test_send_failure_injector.c b/example/urma_performance/test_send_failure_injector.c new file mode 100644 index 0000000000..095ef0a6ff --- /dev/null +++ b/example/urma_performance/test_send_failure_injector.c @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "urma_api.h" + +#ifdef URMA_INJECTOR_TEST_PROVIDER + +static int g_posts; +int UrmaInjectorTestPostCount(void) { return g_posts; } + +urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, + urma_jfs_wr_t** bad_wr) { + (void)jetty; + (void)wr; + if (bad_wr) { + *bad_wr = NULL; + } + ++g_posts; + return URMA_SUCCESS; +} + +#else + +int UrmaInjectorTestPostCount(void); +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "Check failed: %s at line %d\n", #expr, __LINE__); \ + return 1; \ + } \ + } while (0) + +int main(void) { + char marker[] = "/tmp/brpc-urma-injector-XXXXXX"; + int fd = mkstemp(marker); + CHECK(fd >= 0); + close(fd); + CHECK(unlink(marker) == 0); + CHECK(setenv("BRPC_URMA_FAIL_ONCE_FILE", marker, 1) == 0); + urma_jfs_wr_t wr = {0}; + urma_jfs_wr_t* bad = NULL; + wr.opcode = URMA_OPC_SEND; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 1); + + FILE* file = fopen(marker, "w"); + CHECK(file != NULL); + fclose(file); + wr.opcode = URMA_OPC_SEND_IMM; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 2); + CHECK(access(marker, F_OK) == 0); + + wr.opcode = URMA_OPC_SEND; + errno = 0; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == EIO); + CHECK(errno == EIO && bad == &wr && UrmaInjectorTestPostCount() == 2); + CHECK(access(marker, F_OK) != 0); + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 3); + + file = fopen(marker, "w"); + CHECK(file != NULL); + fclose(file); + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 4); + CHECK(unlink(marker) == 0); + puts( + "PASS: forwarding, credit ACK, single failure, and resumed forwarding"); + return 0; +} + +#endif diff --git a/src/brpc/acceptor.cpp b/src/brpc/acceptor.cpp index 99280037d5..36a3f3e827 100644 --- a/src/brpc/acceptor.cpp +++ b/src/brpc/acceptor.cpp @@ -338,6 +338,7 @@ void Acceptor::OnNewConnectionsUntilEAGAIN(Socket* acception) { options.force_ssl = am->_force_ssl; options.initial_ssl_ctx = am->_ssl_ctx; options.socket_mode = am->_socket_mode; + options.urma_qos = am->_urma_qos; options.bthread_tag = am->_bthread_tag; if (Socket::Create(options, &socket_id) != 0) { am->ReleaseConnectionSlot(); diff --git a/src/brpc/acceptor.h b/src/brpc/acceptor.h index 01a2b26a9d..661994f528 100644 --- a/src/brpc/acceptor.h +++ b/src/brpc/acceptor.h @@ -24,7 +24,7 @@ #include "butil/containers/flat_map.h" #include "brpc/input_messenger.h" #include "brpc/socket_mode.h" - +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -137,6 +137,7 @@ friend class Server; // Choose to use a certain socket: 0 TCP, 1 RDMA SocketMode _socket_mode; + UrmaQosOptions _urma_qos; // Acceptor belongs to this tag bthread_tag_t _bthread_tag; diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index 7b5f0a0da6..4f3cf6ab85 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -78,11 +78,9 @@ ChannelSSLOptions* ChannelOptions::mutable_ssl_options() { } static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { - if (opt.auth == nullptr && - !opt.has_ssl_options() && - opt.client_host.empty() && - opt.device_name.empty() && - opt.connection_group.empty() && + if (opt.socket_mode != SOCKET_MODE_URMA && opt.auth == nullptr && + !opt.has_ssl_options() && opt.client_host.empty() && + opt.device_name.empty() && opt.connection_group.empty() && opt.hc_option.health_check_path.empty()) { // Returning zeroized result by default is more intuitive for users. return ChannelSignature(); @@ -141,6 +139,10 @@ static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { buf.append("|rdma"); } else if (opt.socket_mode == SOCKET_MODE_URMA) { buf.append("|urma"); + buf.append("|priority=") + .append(std::to_string(opt.urma_qos.priority)); + buf.append("|sl=").append( + std::to_string(opt.urma_qos.service_level)); } butil::MurmurHash3_x64_128_Update(&mm_ctx, buf.data(), buf.size()); buf.clear(); @@ -187,6 +189,12 @@ int Channel::InitChannelOptions(const ChannelOptions* options) { if (options) { // Override default options if user provided one. _options = *options; } + if (_options.socket_mode == SOCKET_MODE_URMA && + !_options.urma_qos.IsValid()) { + LOG(ERROR) << "URMA priority must be -1 or in [0,15], " + "and SL must be -1 or in [0,255]"; + return -1; + } const Protocol* protocol = FindProtocol(_options.protocol); if (nullptr == protocol || !protocol->support_client()) { LOG(ERROR) << "Channel does not support the protocol"; @@ -407,6 +415,7 @@ int Channel::InitSingle(const butil::EndPoint& server_addr_and_port, opt.local_side = client_endpoint; opt.initial_ssl_ctx = ssl_ctx; opt.socket_mode = _options.socket_mode; + opt.urma_qos = _options.urma_qos; opt.hc_option = _options.hc_option; opt.device_name = _options.device_name; if (SocketMapInsert(SocketMapKey(server_addr_and_port, sig), @@ -453,6 +462,7 @@ int Channel::Init(const char* ns_url, ns_opt.succeed_without_server = _options.succeed_without_server; ns_opt.log_succeed_without_server = _options.log_succeed_without_server; ns_opt.socket_option.socket_mode = _options.socket_mode; + ns_opt.socket_option.urma_qos = _options.urma_qos; ns_opt.channel_signature = ComputeChannelSignature(_options); ns_opt.socket_option.hc_option = _options.hc_option; ns_opt.socket_option.local_side = client_endpoint; diff --git a/src/brpc/channel.h b/src/brpc/channel.h index a778a1bd2a..e295b5275f 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -38,6 +38,7 @@ #include "brpc/naming_service_filter.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -111,6 +112,9 @@ struct ChannelOptions { // Default: SOCKET_MODE_TCP SocketMode socket_mode; + // Outbound URMA QoS for this Channel. Used when socket_mode is URMA. + UrmaQosOptions urma_qos; + // Turn on authentication for this channel if `auth' is not nullptr. // Note `auth' will not be deleted by channel and must remain valid when // the channel is being used. diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 3dcacb96f8..1a70543d18 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -876,6 +876,13 @@ int Server::StartInternal(const butil::EndPoint& endpoint, const ServerOptions default_opt; const ServerOptions& real_opt = opt ? *opt : default_opt; + if (real_opt.socket_mode == SOCKET_MODE_URMA && + !real_opt.urma_qos.IsValid()) { + LOG(ERROR) << "URMA priority must be -1 or in [0,15], " + "and SL must be -1 or in [0,255]"; + return -1; + } + if (!real_opt.h2_settings.IsValid(true/*log_error*/)) { LOG(ERROR) << "Invalid h2_settings"; return -1; @@ -1163,6 +1170,7 @@ int Server::StartInternal(const butil::EndPoint& endpoint, _am->_socket_mode = _options.socket_mode; _am->_bthread_tag = _options.bthread_tag; } + _am->_urma_qos = _options.urma_qos; // Set `_status' to RUNNING before accepting connections // to prevent requests being rejected as ELOGOFF _status = RUNNING; diff --git a/src/brpc/server.h b/src/brpc/server.h index 31dc69af2f..130d181a9c 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -46,6 +46,7 @@ #include "brpc/baidu_master_service.h" #include "brpc/rpc_pb_message_factory.h" #include "brpc/socket_mode.h" +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -242,6 +243,9 @@ struct ServerOptions { // Default: SOCKET_MODE_TCP SocketMode socket_mode; + // Outbound URMA QoS for responses on accepted connections. + UrmaQosOptions urma_qos; + // [CAUTION] This option is for implementing specialized baidu-std proxies, // most users don't need it. Don't change this option unless you fully // understand the description below. diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp index c4411c9606..856fb9ec48 100644 --- a/src/brpc/socket.cpp +++ b/src/brpc/socket.cpp @@ -711,6 +711,10 @@ void SetSocketBufferOptions(int fd) { // version: from version part of _versioned_nref, must be an EVEN number. // slot: designated by ResourcePool. int Socket::Create(const SocketOptions& options, SocketId* id) { + if (options.socket_mode == SOCKET_MODE_URMA && !options.urma_qos.IsValid()) { + errno = EINVAL; + return -1; + } return VersionedRefWithId::Create(id, options); } @@ -726,6 +730,7 @@ int Socket::OnCreated(const SocketOptions& options) { }); // start build the transport _socket_mode = options.socket_mode; + _urma_qos = options.urma_qos; _transport = TransportFactory::CreateTransport(options.socket_mode); CHECK(nullptr != _transport); _transport->Init(this, options); @@ -2814,6 +2819,7 @@ SocketOptions Socket::GetSubSocketOptions() const { opt.keytable_pool = _keytable_pool; opt.app_connect = _app_connect; opt.socket_mode = _socket_mode; + opt.urma_qos = _urma_qos; opt.keepalive_options = _keepalive_options; opt.tcp_user_timeout_ms = _tcp_user_timeout_ms; opt.bthread_tag = _io_event.bthread_tag(); diff --git a/src/brpc/socket.h b/src/brpc/socket.h index 8a1d372294..a17eef3265 100644 --- a/src/brpc/socket.h +++ b/src/brpc/socket.h @@ -44,6 +44,7 @@ #include "brpc/versioned_ref_with_id.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#include "brpc/urma/urma_qos.h" #include "brpc/input_messenger_processor.h" // InputMessengerProcessor namespace brpc { @@ -310,6 +311,7 @@ struct SocketOptions { bool force_ssl{false}; std::shared_ptr initial_ssl_ctx; SocketMode socket_mode{SOCKET_MODE_TCP}; + UrmaQosOptions urma_qos; bthread_keytable_pool_t* keytable_pool{nullptr}; SocketConnection* conn{nullptr}; std::shared_ptr app_connect; @@ -640,6 +642,7 @@ friend class TransportFactory; // The socket mode decided at creation. SocketMode socket_mode() const { return _socket_mode; } + const UrmaQosOptions& urma_qos() const { return _urma_qos; } // Get an UNUSED socket connecting to the same place as this socket // from the SocketPool of this socket. @@ -1017,6 +1020,7 @@ friend class TransportFactory; // Should use SOCKET_MODE_RDMA or SOCKET_MODE_TCP or Other, default is SOCKET_MODE_TCP Transport SocketMode _socket_mode; + UrmaQosOptions _urma_qos; std::unique_ptr _transport; // Pass from controller, for progressive reading. diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index b0572c2e86..55129e34f2 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -240,10 +240,20 @@ urma_status_t urma_query_device(urma_device_t *device, if (!device || !attr) { return URMA_EINVAL; } + mock_device_attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[3].SL = 3; + mock_device_attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[6].SL = 2; + mock_device_attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[10].SL = 4; mock_device_attr.dev_cap.max_jfc = 1024; mock_device_attr.dev_cap.max_jetty = 1024; mock_device_attr.dev_cap.max_jfs_sge = 8; mock_device_attr.dev_cap.max_jfr_sge = 8; + mock_device_attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[0].SL = 2; memcpy(attr, &mock_device_attr, sizeof(urma_device_attr_t)); return URMA_SUCCESS; } @@ -390,13 +400,19 @@ urma_target_seg_t *urma_register_seg(urma_context_t *ctx, urma_seg_cfg_t *cfg) { if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) { return nullptr; } - urma_target_seg_t *seg = new urma_target_seg_t; + urma_target_seg_t *seg = new urma_target_seg_t{}; memset(&seg->seg.ubva.eid, 0, sizeof(urma_eid_t)); seg->seg.ubva.eid.raw[0] = 1; seg->seg.ubva.uasid = 0; seg->seg.ubva.va = cfg->va; seg->seg.len = cfg->len; seg->seg.token_id = cfg->token_value.token; + seg->seg.attr.bs.access = cfg->flag.bs.access; + seg->seg.attr.bs.token_policy = cfg->flag.bs.token_policy; + seg->seg.attr.bs.cacheable = cfg->flag.bs.cacheable; + seg->user_ctx = cfg->user_ctx; + seg->urma_ctx = ctx; + seg->token_id = cfg->token_id; seg_map[seg] = 1; return seg; } @@ -533,89 +549,81 @@ urma_status_t urma_modify_jetty(urma_jetty_t *jetty, urma_jetty_attr_t *attr) { urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr) { - std::shared_lock read_lock(g_rw_mutex); - auto local_it = jetty_map.find(jetty); - auto local_jfc_it = - jetty ? jfc_state_map.find(jetty->jetty_cfg.jfs_cfg.jfc) - : jfc_state_map.end(); - if (!jetty || !wr || local_it == jetty_map.end() || - local_jfc_it == jfc_state_map.end()) { + // Protect the target, pending receive buffer and both completion queues + // from teardown until delivery and notification are complete. + std::unique_lock lock(g_rw_mutex); + if (!jetty || !wr || jetty_map.find(jetty) == jetty_map.end()) { + if (bad_wr) { + *bad_wr = wr; + } + return URMA_EINVAL; + } + urma_jfc_t* local_jfc = jetty->jetty_cfg.jfs_cfg.jfc; + const auto local_state = jfc_state_map.find(local_jfc); + if (local_state == jfc_state_map.end()) { if (bad_wr) { *bad_wr = wr; } return URMA_EINVAL; } - JfcState* local_state = local_jfc_it->second; - read_lock.unlock(); for (urma_jfs_wr_t* current = wr; current; current = current->next) { - if (!current->tjetty) { - if (bad_wr) { - *bad_wr = current; - } + if (bad_wr) { + *bad_wr = current; + } + if (target_jetty_map.find(current->tjetty) == target_jetty_map.end()) { return URMA_EINVAL; } - - urma_cr_t send_cr{}; - send_cr.status = URMA_CR_SUCCESS; - send_cr.user_ctx = current->user_ctx; - send_cr.flag.bs.s_r = 0; - if (current->flag.bs.complete_enable) { - PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, - local_state, send_cr); + const auto remote = jetty_id_map.find(current->tjetty->id.id); + if (remote == jetty_id_map.end()) { + return URMA_FAIL; } - - PendingRecv recv{}; - urma_jfc_t* remote_jfc = nullptr; - { - std::unique_lock lock(g_rw_mutex); - auto remote_it = jetty_id_map.find(current->tjetty->id.id); - if (remote_it == jetty_id_map.end()) { - continue; - } - urma_jfr_t* remote_jfr = - remote_it->second->jetty_cfg.shared.jfr; - auto recv_it = jfr_recv_map.find(remote_jfr); - auto jfc_it = jfr_jfc_map.find(remote_jfr); - if (recv_it == jfr_recv_map.end() || recv_it->second.empty() || - jfc_it == jfr_jfc_map.end()) { - continue; - } - recv = recv_it->second.front(); - recv_it->second.pop_front(); - remote_jfc = jfc_it->second; + urma_jfr_t* jfr = remote->second->jetty_cfg.shared.jfr; + const auto pending = jfr_recv_map.find(jfr); + const auto cq = jfr_jfc_map.find(jfr); + if (pending == jfr_recv_map.end() || cq == jfr_jfc_map.end()) { + return URMA_FAIL; + } + const auto remote_state = jfc_state_map.find(cq->second); + if (remote_state == jfc_state_map.end()) { + return URMA_FAIL; + } + if (pending->second.empty()) { + return URMA_EAGAIN; + } + const PendingRecv recv = pending->second.front(); + const auto& src = current->send.src; + uint64_t length = 0; + for (uint32_t i = 0; i < src.num_sge; ++i) { + length += src.sge[i].len; } + if (length > recv.len) { + return URMA_EINVAL; + } + pending->second.pop_front(); uint32_t copied = 0; - for (uint32_t i = 0; - i < current->send.src.num_sge && copied < recv.len; ++i) { - const urma_sge_t& sge = current->send.src.sge[i]; - const uint32_t n = std::min(sge.len, recv.len - copied); + for (uint32_t i = 0; i < src.num_sge; ++i) { + const auto& sge = src.sge[i]; std::memcpy(reinterpret_cast(recv.addr + copied), - reinterpret_cast(sge.addr), n); - copied += n; - } - - JfcState* remote_state = nullptr; - { - std::shared_lock lock(g_rw_mutex); - auto state_it = jfc_state_map.find(remote_jfc); - if (state_it != jfc_state_map.end()) { - remote_state = state_it->second; - } + reinterpret_cast(sge.addr), sge.len); + copied += sge.len; } - if (remote_state) { - urma_cr_t recv_cr{}; - recv_cr.status = URMA_CR_SUCCESS; - recv_cr.user_ctx = recv.user_ctx; - recv_cr.flag.bs.s_r = 1; - recv_cr.completion_len = copied; - recv_cr.opcode = - current->opcode == URMA_OPC_SEND_IMM - ? URMA_CR_OPC_SEND_WITH_IMM - : URMA_CR_OPC_SEND; - recv_cr.imm_data = current->send.imm_data; - PushCompletion(remote_jfc, remote_state, recv_cr); + urma_cr_t recv_cr{}; + recv_cr.status = URMA_CR_SUCCESS; + recv_cr.user_ctx = recv.user_ctx; + recv_cr.flag.bs.s_r = 1; + recv_cr.completion_len = copied; + recv_cr.opcode = current->opcode == URMA_OPC_SEND_IMM + ? URMA_CR_OPC_SEND_WITH_IMM + : URMA_CR_OPC_SEND; + recv_cr.imm_data = current->send.imm_data; + PushCompletion(cq->second, remote_state->second, recv_cr); + if (current->flag.bs.complete_enable) { + urma_cr_t send_cr{}; + send_cr.status = URMA_CR_SUCCESS; + send_cr.user_ctx = current->user_ctx; + PushCompletion(local_jfc, local_state->second, send_cr); } } if (bad_wr) { diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 26c7a1b9dc..54d7fa0aa4 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -19,6 +19,7 @@ #if BRPC_WITH_URMA +#include #include #include @@ -35,6 +36,7 @@ #include #include "butil/atomicops.h" +#include "butil/fd_guard.h" #include "butil/iobuf.h" #include "butil/logging.h" #include "butil/macros.h" @@ -64,6 +66,7 @@ DECLARE_int32(urma_sq_size); DECLARE_int32(urma_rq_size); DECLARE_int32(urma_cqe_poll_once); DECLARE_bool(urma_recv_zerocopy); +DECLARE_bool(urma_copy_send); DECLARE_int32(urma_zerocopy_min_size); DECLARE_int32(urma_prepared_jetty_cnt); DECLARE_bool(urma_poller_yield); @@ -96,11 +99,12 @@ static int PreparedJettyCount() { } // In event mode each prepared JFCE consumes a file descriptor. Keep room - // for one TCP fd per future URMA connection and for brpc/system internals. + // for a TCP fd and a CQ carrier fd per future URMA connection, plus + // descriptors used by brpc/system internals. static const rlim_t kReservedFdCount = 64; const rlim_t max_prepared = nofile.rlim_cur > kReservedFdCount - ? (nofile.rlim_cur - kReservedFdCount) / 2 + ? (nofile.rlim_cur - kReservedFdCount) / 3 : 0; if (max_prepared >= static_cast(requested)) { return requested; @@ -168,6 +172,9 @@ UrmaEndpoint::~UrmaEndpoint() { void UrmaEndpoint::Reset() { DeallocateResources(); _state = UNINIT; + _recovery_epoch = 0; + _runtime_failure.store(UrmaFailureReason::NONE, + butil::memory_order_relaxed); _handshake_version = 0; _remote_recv_block_size = 0; _local_window_capacity = 0; @@ -181,9 +188,6 @@ void UrmaEndpoint::Reset() { _rq_received = 0; _pending_received_bytes.store(0, butil::memory_order_relaxed); _input_processor.Reset(); - _sbuf.clear(); - _rbuf.clear(); - _rbuf_data.clear(); _read_butex->store(0, butil::memory_order_relaxed); } @@ -262,7 +266,7 @@ void UrmaEndpoint::MakeLocalParsedHello(ParsedHello* out) const { : _resource->jetty->jetty_id.eid.raw; std::memcpy(out->eid, advertised_eid, 16); } - out->tp_type = static_cast(URMA_CTP); + out->tp_type = static_cast(GetUrmaTpType()); // Pool segment: flatten g_pool_seg's seg fields. urma_target_seg_t* pool = GetPoolSegFor(nullptr); if (pool) { @@ -386,10 +390,28 @@ int UrmaEndpoint::ReadAndParseHelloV3(ParsedHello* out, bool* negotiated) { // Allocate / deallocate per-connection resources. // ============================================================================ +namespace { + +class UrmaCqSocketUser : public SocketUser { +public: + explicit UrmaCqSocketUser(SocketId owner) : owner_id(owner) {} + + void BeforeRecycle(Socket*) override { delete this; } + + const SocketId owner_id; +}; + +} // namespace + int UrmaEndpoint::AllocateResources() { + std::unique_lock lock(_cq_mutex); if (_resource) { return 0; } + uint8_t priority; + if (!ResolveUrmaJettyPriority(_socket->urma_qos(), &priority)) { + return -1; + } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { errno = ENODEV; @@ -402,7 +424,8 @@ int UrmaEndpoint::AllocateResources() { } // Try the prepared pool first (sized sq/rq match). - if (_sq_size <= static_cast(FLAGS_urma_sq_size) && + if (priority == GetUrmaJettyPriority() && + _sq_size <= static_cast(FLAGS_urma_sq_size) && _rq_size <= static_cast(FLAGS_urma_rq_size)) { BAIDU_SCOPED_LOCK(g_prepared_mutex); if (g_prepared_list) { @@ -451,7 +474,7 @@ int UrmaEndpoint::AllocateResources() { jetty_cfg.flag.bs.share_jfr = 1; jetty_cfg.jfs_cfg.depth = static_cast(_sq_size); jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; - jetty_cfg.jfs_cfg.priority = GetUrmaJettyPriority(); + jetty_cfg.jfs_cfg.priority = priority; jetty_cfg.jfs_cfg.max_sge = static_cast(GetUrmaMaxSge()); jetty_cfg.jfs_cfg.rnr_retry = URMA_TYPICAL_RNR_RETRY; @@ -470,7 +493,9 @@ int UrmaEndpoint::AllocateResources() { _rbuf.resize(_rq_size); _rbuf_data.resize(_rq_size, nullptr); - // Wrap the JFCE fd in a brpc Socket so PollCq is driven by epoll. + // The provider owns the original JFCE fd. Give the carrier its own + // descriptor so its lifetime does not depend on provider teardown. + butil::fd_guard event_fd; if (!FLAGS_urma_use_polling) { if (!_resource->jfce || _resource->jfce->fd < 0) { LOG(ERROR) << "Prepared URMA resource has no usable JFCE"; @@ -480,58 +505,59 @@ int UrmaEndpoint::AllocateResources() { if (ReqNotifyCq() != 0) { return -1; } - SocketOptions options; - options.user = this; - options.keytable_pool = _socket->keytable_pool(); - options.fd = _resource->jfce->fd; - options.on_edge_triggered_events = PollCq; - if (Socket::Create(options, &_cq_sid) < 0) { - PLOG(ERROR) << "Fail to create CQ socket"; + event_fd.reset(fcntl(_resource->jfce->fd, F_DUPFD_CLOEXEC, 0)); + if (event_fd < 0) { return -1; } - } else { - // Polling mode: synthetic carrier socket (no fd). - SocketOptions options; - options.user = this; - options.keytable_pool = _socket->keytable_pool(); - options.on_edge_triggered_events = PollCq; - if (Socket::Create(options, &_cq_sid) < 0) { - PLOG(ERROR) << "Fail to create CQ socket (polling)"; + } + std::unique_ptr user( + new (std::nothrow) UrmaCqSocketUser(_socket->id())); + if (!user) { + errno = ENOMEM; + return -1; + } + SocketOptions options; + options.keytable_pool = _socket->keytable_pool(); + options.on_edge_triggered_events = PollCq; + if (Socket::Create(options, &_cq_sid) != 0) { + PLOG(ERROR) << "Fail to create CQ socket"; + return -1; + } + // Install the owner before publishing the carrier to epoll or the poller. + SocketUniquePtr carrier; + CHECK_EQ(0, Socket::Address(_cq_sid, &carrier)); + carrier->_user = user.release(); + if (event_fd >= 0) { + if (carrier->ResetFileDescriptor(event_fd) != 0) { return -1; } + event_fd.release(); + } else { PollerAddCqSid(); } + _effective_priority.store(priority, butil::memory_order_relaxed); return 0; } void UrmaEndpoint::DeallocateResources() { - if (!_resource) { - return; - } - + std::unique_lock lock(_cq_mutex); if (FLAGS_urma_use_polling) { PollerRemoveCqSid(); } - // Tear down the CQ socket so the EventDispatcher stops calling PollCq. if (_cq_sid != INVALID_SOCKET_ID) { - SocketUniquePtr s; - if (Socket::Address(_cq_sid, &s) == 0) { - if (s->fd() >= 0) { - s->_io_event.RemoveConsumer(s->_fd); - } - s->_user = nullptr; // Do not release user (this UrmaEndpoint). - s->_fd = -1; // Already removed fd from epoll. - s->SetFailed(); - } + Socket::SetFailed(_cq_sid); _cq_sid = INVALID_SOCKET_ID; } - // Reusing a Jetty requires a driver-supported RESET plus a complete JFC - // drain. Until that lifecycle is implemented, prepared resources are - // one-shot: they accelerate connection setup but are destroyed on close. + // Prepared resources are one-shot. Destroy provider handles before + // releasing buffers referenced by outstanding work requests. + _effective_priority.store(-1, butil::memory_order_relaxed); delete _resource; _resource = nullptr; + _sbuf.clear(); + _rbuf.clear(); + _rbuf_data.clear(); } // ============================================================================ @@ -539,6 +565,12 @@ void UrmaEndpoint::DeallocateResources() { // ============================================================================ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { + if (peer.tp_type != static_cast(GetUrmaTpType())) { + LOG(ERROR) << "URMA peer path mismatch: local=" << GetUrmaTpType() + << " peer=" << static_cast(peer.tp_type); + errno = EPROTO; + return -1; + } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { errno = ENODEV; @@ -572,10 +604,6 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { remote.jetty_id.id = peer.jetty_id; remote.trans_mode = URMA_TM_RM; remote.type = URMA_JETTY; - if (peer.tp_type > static_cast(URMA_UTP)) { - errno = EPROTO; - return -1; - } remote.tp_type = static_cast(peer.tp_type); urma_token_t token{}; @@ -629,29 +657,20 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { class UrmaIOBuf : private butil::IOBuf { friend class ::brpc::urma::UrmaEndpoint; public: - using butil::IOBuf::_ref_num; - using butil::IOBuf::_ref_at; - using butil::IOBuf::fetch1; - using butil::IOBuf::get_first_data_meta; - using butil::IOBuf::cutn; - // Build the SGE for the current head block. - // Returns bytes added, or -1 (errno set). - ssize_t cut_into_sglist(urma_sge_t* sglist, size_t* sge_index, - butil::IOBuf* to, size_t max_sge, - size_t max_len) { + // Describe the prefix without consuming it. The writer commits only + // after the provider accepts the WR. + ssize_t DescribePrefix(urma_sge_t* sglist, size_t* sge_index, + size_t max_sge, size_t max_len) const { size_t len = 0; - while (*sge_index < max_sge && len < max_len && _ref_num() != 0) { - butil::IOBuf::BlockRef const& r = _ref_at(0); - const void* start = fetch1(); + for (size_t i = 0; + i < _ref_num() && *sge_index < max_sge && len < max_len; ++i) { + const butil::IOBuf::BlockRef& r = _ref_at(i); + void* start = r.block->data + r.offset; urma_target_seg_t* tseg = - GetPoolSegFor(const_cast(start)); - if (!tseg) { - // User-registered memory: look up the seg handle. - uint64_t meta = get_first_data_meta(); - if (meta != 0) { - tseg = reinterpret_cast( - static_cast(meta)); - } + GetPoolSegFor(start); + if (!tseg && r.block->is_user_data()) { + tseg = reinterpret_cast( + static_cast(r.block->u.data_meta)); } if (!tseg) { errno = ERDMAMEM; @@ -661,10 +680,18 @@ class UrmaIOBuf : private butil::IOBuf { if (len + this_len > max_len) { this_len = max_len - len; } - sglist[*sge_index].addr = reinterpret_cast(start); - sglist[*sge_index].len = static_cast(this_len); - sglist[*sge_index].tseg = tseg; - cutn(to, this_len); + if (!InitUrmaLocalSge(&sglist[*sge_index], + reinterpret_cast(start), this_len, + tseg)) { + LOG(ERROR) << "Invalid URMA send SGE: address=" + << reinterpret_cast(start) + << " length=" << this_len + << " segment_va=" << tseg->seg.ubva.va + << " segment_len=" << tseg->seg.len + << " context_matches=" + << (tseg->urma_ctx == GetUrmaContext()); + return -1; + } len += this_len; (*sge_index)++; } @@ -677,6 +704,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { errno = ENOTCONN; return -1; } + const bool copy_send = FLAGS_urma_copy_send; int max_sge = GetUrmaMaxSge(); if (max_sge < 1) { max_sge = 1; @@ -693,7 +721,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ssize_t total_len = 0; while (current < ndata) { uint16_t remote_wnd = _remote_rq_window_size.load(butil::memory_order_relaxed); - uint16_t sq_wnd = _sq_window_size.load(butil::memory_order_relaxed); + uint16_t sq_wnd = _sq_window_size.load(butil::memory_order_acquire); if (remote_wnd == 0 || sq_wnd == 0) { if (total_len > 0) { break; @@ -707,23 +735,43 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { size_t max_len = _remote_recv_block_size > 0 ? _remote_recv_block_size : GetUrmaRecvBlockSize(); - while (sge_index < static_cast(max_sge) && - this_len < max_len && current < ndata) { - auto* data = reinterpret_cast(from[current]); - if (data->empty()) { - ++current; - continue; - } - ssize_t n = data->cut_into_sglist(sglist, &sge_index, to, - max_sge, max_len - this_len); + if (copy_send) { + max_len = std::min(max_len, GetUrmaRecvBlockSize() - + (URMA_SEND_BUFFER_ALIGNMENT - 1)); + } + for (size_t i = current; + i < ndata && this_len < max_len && + sge_index < static_cast(max_sge); ++i) { + const auto* data = reinterpret_cast(from[i]); + const ssize_t n = + copy_send ? static_cast( + std::min(from[i]->size(), max_len - this_len)) + : data->DescribePrefix(sglist, &sge_index, max_sge, + max_len - this_len); if (n < 0) { + to->clear(); + if (errno == ERDMAMEM) { + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + } return -1; } + from[i]->append_to(to, n); this_len += n; } - if (sge_index == 0) { + if (this_len == 0) { break; } + if (copy_send) { + if (!CopyUrmaSendBuffer(to, &sglist[0])) { + LOG(ERROR) + << "Fail to prepare aligned URMA send buffer: length=" + << this_len << " errno=" << errno; + to->clear(); + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + return -1; + } + sge_index = 1; + } urma_sg_t sg{sglist, static_cast(sge_index)}; urma_jfs_wr_t wr{}; @@ -736,7 +784,8 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { wr.flag.bs.complete_enable = 1; wr.tjetty = _resource->remote_jetty; wr.send.src = sg; - wr.user_ctx = 1; + // Zero identifies credit ACKs; data completions identify their SQ slot. + wr.user_ctx = static_cast(_sq_current) + 1; urma_jfs_wr_t* bad_wr = nullptr; const uint16_t sq_slot = _sq_current; const uint32_t local_jetty_id = _resource->jetty->jetty_id.id; @@ -755,6 +804,11 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { const int provider_errno = errno; _remote_rq_window_size.fetch_add(1, butil::memory_order_relaxed); _sq_window_size.fetch_add(1, butil::memory_order_relaxed); + to->clear(); + if (rc == URMA_EAGAIN) { + errno = EAGAIN; + return total_len ? total_len : -1; + } LOG(WARNING) << "urma_post_jetty_send_wr failed: " << rc << ", provider_errno=" << provider_errno << " (" << berror(provider_errno) << ')' @@ -770,9 +824,17 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { << ", configured_max_sge=" << GetUrmaMaxSge() << ", payload_size=" << this_len << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = rc; return -1; } + // Commit the accepted prefix across the original write requests. + for (size_t remaining = this_len; remaining > 0;) { + remaining -= from[current]->pop_front(remaining); + if (from[current]->empty()) { + ++current; + } + } _sq_current = (_sq_current + 1) % (_sq_size - RESERVED_WR_NUM); total_len += static_cast(this_len); } @@ -797,7 +859,8 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { urma_sge_t sge{reinterpret_cast(block), static_cast(block_size), tseg, nullptr}; urma_sg_t sg{&sge, 1}; - urma_jfr_wr_t wr{sg, 0, nullptr}; + // Preserve the receive address in completions for access-error diagnostics. + urma_jfr_wr_t wr{sg, reinterpret_cast(block), nullptr}; urma_jfr_wr_t* bad = nullptr; // Use the shared-JFR path on every device, including bonding. The bonding // provider owns physical receive scheduling for the JFR; a local @@ -815,6 +878,7 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { _resource->jetty->remote_jetty) << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::RECV_POST); errno = status; return -1; } @@ -908,6 +972,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { << _resource->remote_jetty->id.id << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = status; return -1; } @@ -927,7 +992,74 @@ int UrmaEndpoint::SendAck(int num) { ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { - LOG(WARNING) << "URMA completion failed, status=" << cr.status; + LOG(WARNING) << "URMA completion failed, status=" << cr.status + << " direction=" << (cr.flag.bs.s_r ? "recv" : "send") + << " urma_priority=" << _socket->urma_qos().priority + << " urma_sl=" << _socket->urma_qos().service_level + << " urma_effective_priority=" + << (_resource && _resource->jetty + ? static_cast( + _resource->jetty->jetty_cfg.jfs_cfg.priority) + : -1) + << " urma_effective_sl=" + << (_resource && _resource->jetty + ? GetUrmaServiceLevelForPriority( + _resource->jetty->jetty_cfg.jfs_cfg.priority) + : -1) + << " flags=" << static_cast(cr.flag.value) + << " opcode=" << cr.opcode << " local_id=" << cr.local_id + << " tpn=" << cr.tpn << " user_ctx=" << cr.user_ctx + << " completion_len=" << cr.completion_len << " jetty_id=" + << (_resource && _resource->jetty + ? _resource->jetty->jetty_id.id + : 0) + << " jfr_id=" + << (_resource && _resource->jfr ? _resource->jfr->jfr_id.id + : 0) + << " state=" << GetStateStr() << " on " + << _socket->description(); + if (!cr.flag.bs.s_r && cr.user_ctx > 0 && cr.user_ctx <= _sbuf.size()) { + const size_t slot = static_cast(cr.user_ctx - 1); + const auto& buffer = _sbuf[slot]; + LOG(WARNING) << "URMA failed SEND buffer: slot=" << slot + << " bytes=" << buffer.size() + << " blocks=" << buffer.backing_block_num() << " on " + << _socket->description(); + for (size_t i = 0; i < buffer.backing_block_num(); ++i) { + const auto block = buffer.backing_block(i); + auto* segment = GetPoolSegFor(const_cast(block.data())); + LOG(WARNING) + << "URMA failed SEND block: slot=" << slot << " block=" << i + << " address=" << reinterpret_cast(block.data()) + << " length=" << block.size() + << " pool_segment=" << (segment != nullptr) + << " segment_va=" << (segment ? segment->seg.ubva.va : 0) + << " segment_len=" << (segment ? segment->seg.len : 0) + << " segment_token_id=" + << (segment ? segment->seg.token_id : 0) << " on " + << _socket->description(); + } + } + switch (cr.status) { + case URMA_CR_ACK_TIMEOUT_ERR: + ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); + break; + case URMA_CR_RNR_RETRY_CNT_EXC_ERR: + ReportFailure(UrmaFailureReason::RNR); + break; + case URMA_CR_LOC_ACCESS_ERR: + case URMA_CR_REM_ACCESS_ABORT_ERR: + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + break; + case URMA_CR_WR_FLUSH_ERR: + case URMA_CR_WR_FLUSH_ERR_DONE: + case URMA_CR_WR_SUSPEND_DONE: + case URMA_CR_WR_UNHANDLED: + break; + default: + ReportFailure(UrmaFailureReason::TRANSPORT_ERROR); + break; + } errno = EIO; return -1; } @@ -946,34 +1078,21 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { return -1; } _sq_imm_window_size += 1; - SendAck(0); - return 0; - } - uint16_t wnd = 1; // We signal every WR (complete_enable=1). - uint16_t old = - _sq_window_size.load(butil::memory_order_relaxed); - while (true) { - if (old >= _local_window_capacity) { - LOG(WARNING) - << "URMA send completion exceeds SQ window: old=" << old - << " increment=" << wnd - << " capacity=" << _local_window_capacity - << " user_ctx=" << cr.user_ctx - << " on " << _socket->description(); - errno = EPROTO; - return -1; - } - if (_sq_window_size.compare_exchange_weak( - old, static_cast(old + wnd), - butil::memory_order_relaxed)) { - break; - } + return SendAck(0); } - for (uint16_t i = 0; i < wnd; ++i) { - _sbuf[_sq_sent].clear(); - _sq_sent = (_sq_sent + 1) % (_sq_size - RESERVED_WR_NUM); + const uint16_t old = _sq_window_size.load(butil::memory_order_relaxed); + if (old >= _local_window_capacity) { + LOG(WARNING) << "URMA send completion exceeds SQ window: old=" << old + << " capacity=" << _local_window_capacity + << " user_ctx=" << cr.user_ctx + << " on " << _socket->description(); + errno = EPROTO; + return -1; } - butil::subtle::MemoryBarrier(); + // Reclaim the buffer before making its slot available to the writer. + _sbuf[_sq_sent].clear(); + _sq_sent = (_sq_sent + 1) % (_sq_size - RESERVED_WR_NUM); + _sq_window_size.fetch_add(1, butil::memory_order_release); if (_remote_rq_window_size.load(butil::memory_order_relaxed) >= _local_window_capacity / 8) { _socket->WakeAsEpollOut(); @@ -1035,8 +1154,8 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (cr.completion_len > 0) { - SendAck(1); + if (cr.completion_len > 0 && SendAck(1) < 0) { + return -1; } return static_cast(cr.completion_len); } @@ -1080,24 +1199,31 @@ void UrmaEndpoint::DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes) { } void UrmaEndpoint::PollCq(Socket* m) { - auto* ep = static_cast(m->user()); - if (!ep || !ep->_resource || !ep->_resource->jfc) { - return; - } - SocketUniquePtr s; - if (Socket::Address(ep->_socket->id(), &s) != 0) { + // Acquire the fd published by ResetFileDescriptor in event mode. + if (!FLAGS_urma_use_polling && + m->_fd.load(butil::memory_order_acquire) < 0) { return; } - if (s->Failed()) { + const auto* user = static_cast(m->user()); + SocketUniquePtr owner; + if (!user || Socket::Address(user->owner_id, &owner) != 0) { return; } + auto* transport = static_cast(owner->_transport.get()); + transport->_urma_ep->DrainCq(m, owner); +} +void UrmaEndpoint::DrainCq(Socket* m, SocketUniquePtr& s) { const bool event_mode = !FLAGS_urma_use_polling; int progress = Socket::PROGRESS_INIT; while (true) { + std::unique_lock lock(_cq_mutex); + if (m->id() != _cq_sid || !_resource || !_resource->jfc || s->Failed()) { + return; + } urma_jfc_t* event_jfc = nullptr; if (event_mode) { - const int event_count = ep->WaitCqEvent(s, &event_jfc); + const int event_count = WaitCqEvent(s, &event_jfc); if (event_count < 0) { return; } @@ -1116,8 +1242,9 @@ void UrmaEndpoint::PollCq(Socket* m) { std::max(1, std::min(FLAGS_urma_cqe_poll_once, 32)); urma_cr_t crs[32]; const int cnt = - urma_poll_jfc(ep->_resource->jfc, n, crs); + urma_poll_jfc(_resource->jfc, n, crs); if (cnt < 0) { + ReportFailure(UrmaFailureReason::CQ_POLL); return EIO; } if (cnt == 0) { @@ -1127,7 +1254,7 @@ void UrmaEndpoint::PollCq(Socket* m) { if (s->Failed()) { return ECANCELED; } - const ssize_t nr = ep->HandleCompletion(crs[i]); + const ssize_t nr = HandleCompletion(crs[i]); if (nr < 0) { return errno ? errno : EIO; } @@ -1145,7 +1272,7 @@ void UrmaEndpoint::PollCq(Socket* m) { uint32_t nevents = 1; urma_ack_jfc(&event_jfc, &nevents, 1); if (completion_error == 0) { - if (ep->ReqNotifyCq() != 0) { + if (ReqNotifyCq() != 0) { return; } @@ -1163,7 +1290,8 @@ void UrmaEndpoint::PollCq(Socket* m) { } return; } - ep->DispatchReceivedBytes(s, bytes); + lock.unlock(); + DispatchReceivedBytes(s, bytes); if (!event_mode) { return; @@ -1203,8 +1331,6 @@ void UrmaEndpoint::ApplyRemoteHello(const ParsedHello& remote) { // OnNewDataFromTcp: edge-triggered dispatcher. // ============================================================================ -static void TryReadOnTcpDuringUrmaEst(Socket* socket); - void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { auto* tp = static_cast(m->_transport.get()); if (!tp) { @@ -1228,7 +1354,7 @@ void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { if (!IsUrmaAvailable()) { ep->_state = FALLBACK_TCP; tp->_urma_state = UrmaTransport::URMA_OFF; - InputMessenger::OnNewMessages(m); + ep->TryReadOnTcp(); return; } SocketUniquePtr s; @@ -1244,42 +1370,56 @@ void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { } else { s.release(); } - return; } // Client side: handled by ProcessHandshakeAtClient. - return; } else if (state < ESTABLISHED) { // During handshake: wake the handshake bthread parked in ReadFromFd. ep->_read_butex->fetch_add(1, butil::memory_order_release); bthread::butex_wake(ep->_read_butex); - return; } else if (state == FALLBACK_TCP) { - InputMessenger::OnNewMessages(m); + ep->TryReadOnTcp(); return; } else if (state == ESTABLISHED) { - TryReadOnTcpDuringUrmaEst(m); + ep->DrainEstablishedTcp(); return; } + // Return handshake events to the dispatcher as well. Leaving _nevent + // nonzero suppresses later EOF callbacks and retains server resources + // after a short-connection client closes its TCP fd. if (!m->MoreReadEvents(&progress)) { break; } } } -inline void UrmaEndpoint::TryReadOnTcp() { - if (_state.load(butil::memory_order_acquire) == FALLBACK_TCP) { +void UrmaEndpoint::TryReadOnTcp() { + std::unique_lock lock(_tcp_read_mutex); + if (_state.load(butil::memory_order_acquire) == FALLBACK_TCP && + !_socket->Failed()) { InputMessenger::OnNewMessages(_socket); } } -static void TryReadOnTcpDuringUrmaEst(Socket* socket) { +void UrmaEndpoint::DrainEstablishedTcp() { + std::unique_lock lock(_tcp_read_mutex); + if (_state.load(butil::memory_order_acquire) != ESTABLISHED || + _socket->Failed()) { + return; + } + Socket* socket = _socket; int progress = Socket::PROGRESS_INIT; while (true) { uint8_t byte = 0; const ssize_t nr = read(socket->fd(), &byte, 1); if (nr < 0) { + if (errno == EINTR) { + continue; + } if (errno != EAGAIN) { const int saved_errno = errno; + if (socket->CreatedByConnect()) { + ReportFailure(UrmaFailureReason::TRANSPORT_ERROR); + } socket->SetFailed(saved_errno, "Fail to read URMA TCP fd: %s", berror(saved_errno)); return; @@ -1288,6 +1428,12 @@ static void TryReadOnTcpDuringUrmaEst(Socket* socket) { return; } } else if (nr == 0) { + // A failed server SEND closes the control socket without a local + // completion error at the client. Cool that peer before reconnecting. + // Server-side EOF is normal when a short-connection client closes. + if (socket->CreatedByConnect()) { + ReportFailure(UrmaFailureReason::TRANSPORT_ERROR); + } socket->SetEOF(); return; } else { @@ -1307,6 +1453,33 @@ void UrmaEndpoint::FallbackToTcp(UrmaTransport* transport, bool process_tcp) { } } +bool UrmaEndpoint::AllowNewUrmaConnection(const char* peer) { + if (!IsUrmaAvailable()) { + return false; + } + uint64_t epoch = 0; + const bool allowed = + GlobalUrmaRecoveryPolicy().AllowNewConnection(peer, &epoch); + _recovery_epoch.store(epoch, butil::memory_order_relaxed); + return allowed; +} + +void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { + if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { + return; + } + UrmaFailureReason expected = UrmaFailureReason::NONE; + if (reason == UrmaFailureReason::NONE || + !_runtime_failure.compare_exchange_strong(expected, reason)) { + return; + } + GlobalUrmaRecoveryPolicy().ReportPeerFailure( + _socket->CreatedByConnect() + ? butil::endpoint2str(_socket->remote_side()).c_str() + : "", + _recovery_epoch.load(butil::memory_order_relaxed), reason); +} + void UrmaEndpoint::FailHandshake(UrmaTransport* transport, int error, const char* reason) { LOG(ERROR) << "URMA handshake failed in state=" << GetStateStr() @@ -1333,7 +1506,8 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { SocketUniquePtr s(ep->_socket); auto* tp = static_cast(s->_transport.get()); UrmaConnect::RunGuard guard(static_cast(s->_app_connect.get())); - if (!IsUrmaAvailable()) { + if (!ep->AllowNewUrmaConnection( + butil::endpoint2str(s->remote_side()).c_str())) { ep->FallbackToTcp(tp, true); return nullptr; } @@ -1385,7 +1559,10 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection( + ep->_recovery_epoch.load(butil::memory_order_relaxed)); ep->DispatchReceivedBytes(s, 0); + ep->DrainEstablishedTcp(); return nullptr; } @@ -1420,6 +1597,16 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->FailHandshake(tp, EPROTO, "invalid client hello"); return nullptr; } + if (!ep->AllowNewUrmaConnection("")) { + tp->_urma_state = UrmaTransport::URMA_OFF; + if (hs->SendLocalHello() < 0) { + ep->FailHandshake(tp, errno ? errno : EIO, + "send TCP fallback hello"); + return nullptr; + } + ep->FallbackToTcp(tp, true); + return nullptr; + } ep->_state = S_ALLOC_RES; if (ep->AllocateResources() < 0) { const int saved_errno = errno ? errno : EIO; @@ -1470,7 +1657,10 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection( + ep->_recovery_epoch.load(butil::memory_order_relaxed)); ep->DispatchReceivedBytes(s, 0); + ep->DrainEstablishedTcp(); } else { ep->FallbackToTcp(tp, true); } @@ -1555,11 +1745,19 @@ std::string UrmaEndpoint::GetStateStr() const { } void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { - os << "state=" << GetStateStr() - << " sq_size=" << _sq_size << " rq_size=" << _rq_size + const int priority = _effective_priority.load(butil::memory_order_relaxed); + os << "state=" << GetStateStr() << " sq_size=" << _sq_size + << " rq_size=" << _rq_size << " remote_recv_block_size=" << _remote_recv_block_size << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) - << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " remote_rq_window=" + << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " urma_priority=" << _socket->urma_qos().priority + << " urma_sl=" << _socket->urma_qos().service_level + << " urma_effective_priority=" << priority + << " urma_effective_sl=" << GetUrmaServiceLevelForPriority(priority) + << " recovery_epoch=" << _recovery_epoch.load(butil::memory_order_relaxed) + << " runtime_failure=" << UrmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } @@ -1579,6 +1777,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, const int saved_errno = errno; PLOG(ERROR) << "Fail to wait URMA JFC event from " << s->description(); + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to wait URMA JFC event: %s", berror(saved_errno)); return -1; @@ -1589,6 +1788,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, if (*event_jfc != _resource->jfc) { LOG(ERROR) << "Unexpected URMA JFC event on " << s->description(); errno = EPROTO; + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(EPROTO, "Unexpected URMA JFC event"); return -1; } @@ -1604,6 +1804,7 @@ int UrmaEndpoint::ReqNotifyCq() { if (rc != URMA_SUCCESS) { errno = rc; PLOG(WARNING) << "Fail to rearm URMA JFC"; + ReportFailure(UrmaFailureReason::CQ_NOTIFY); _socket->SetFailed(rc, "Fail to rearm URMA JFC: %s", berror(rc)); return -1; } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index e5aed5ac66..2727ee2508 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -18,6 +18,8 @@ #ifndef BRPC_URMA_ENDPOINT_H #define BRPC_URMA_ENDPOINT_H +#include "brpc/urma/urma_recovery_policy.h" + #include #include #include @@ -100,6 +102,8 @@ struct UrmaResource { // windows, and the registered buffer bookkeeping. Cache-line padded to avoid // false sharing between connections. class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { + friend class UrmaRecoveryTestPeer; + friend class UrmaQosTestPeer; friend class UrmaConnect; friend class Socket; friend class UrmaTransport; @@ -144,7 +148,7 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { static void OnNewDataFromTcp(Socket* m); // CQ completion poller: the edge-triggered callback for the CQ socket - // (whose user is this endpoint). Drains urma_poll_jfc and feeds the + // (whose user holds the owning socket ID). Drains urma_poll_jfc and feeds the // completions into HandleCompletion, then calls InputMessenger. Event mode // also drains the aggregated JFCE until urma_wait_jfc reports no event, // then calls Socket::MoreReadEvents so later JFCE edges can start the @@ -211,6 +215,8 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Jetty, plus the CQ socket). Returns 0 on success. int AllocateResources(); void DeallocateResources(); + bool AllowNewUrmaConnection(const char* peer); + void DrainCq(Socket* carrier, SocketUniquePtr& owner); // Import the peer's jetty and buffer-pool segment. CRITICAL: // urma_import_seg is called BEFORE urma_import_jetty, otherwise the kernel @@ -259,8 +265,10 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { void PollerAddCqSid(); void PollerRemoveCqSid(); - inline void TryReadOnTcp(); + void TryReadOnTcp(); + void DrainEstablishedTcp(); void FallbackToTcp(UrmaTransport* transport, bool process_tcp); + void ReportFailure(UrmaFailureReason reason); void FailHandshake(UrmaTransport* transport, int error, const char* reason); @@ -272,17 +280,26 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Not owned. Socket* _socket; + butil::atomic _recovery_epoch{0}; + butil::atomic _runtime_failure{UrmaFailureReason::NONE}; + // Handshake state. butil::atomic _state{UNINIT}; int _handshake_version{0}; // 0 = unnegotiated; 2 = v2; 3 = v3 butil::atomic _pending_received_bytes{0}; butil::Mutex _dispatch_mutex; + // Serialize TCP parsing / EOF checks across event and handshake bthreads. + butil::Mutex _tcp_read_mutex; // The URMA resources (jetty / jfc / jfr / jfce / imported peer objects). UrmaResource* _resource{nullptr}; + // Diagnostics can read the effective priority during handshake teardown. + butil::atomic _effective_priority{-1}; // The SocketId wrapping the JFCE fd (event mode) or a synthetic carrier // (polling mode). PollCq is the edge-triggered callback on this socket. + // Serializes provider CQ operations with carrier registration and teardown. + butil::Mutex _cq_mutex; SocketId _cq_sid{INVALID_SOCKET_ID}; // ---- Send / recv window bookkeeping ---- diff --git a/src/brpc/urma/urma_handshake.cpp b/src/brpc/urma/urma_handshake.cpp index b7018606e0..ac7eb115ce 100644 --- a/src/brpc/urma/urma_handshake.cpp +++ b/src/brpc/urma/urma_handshake.cpp @@ -328,7 +328,12 @@ int UrmaHandshakeServerV3::ReceiveAndParseRemoteHello(ParsedHello* out, int UrmaHandshakeServerV3::SendLocalHello() { UrmaHello msg; _ep->FillLocalHelloV3(&msg); - // v3 has no zero-out path; the client rejects jetty_id==0 via ValidHello. + auto* tp = static_cast(_ep->_socket->_transport.get()); + if (tp->_urma_state.load(butil::memory_order_acquire) == + UrmaTransport::URMA_OFF) { + msg.set_jetty_id(0); + msg.set_buffer_size(0); + } return _ep->WriteHelloV3(msg); } diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index ded662fb22..5bf2dc2976 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -28,6 +28,8 @@ #include #include +#include +#include #include #include #include @@ -75,11 +77,15 @@ DEFINE_int32(urma_cqe_poll_once, 32, "Max completion entries polled per urma_poll_jfc call"); DEFINE_bool(urma_recv_zerocopy, true, "Use zero-copy for receives larger than --urma_zerocopy_min_size"); +DEFINE_bool(urma_copy_send, false, + "Copy host send payloads into a 64-byte aligned registered buffer"); DEFINE_int32(urma_zerocopy_min_size, 512, "Receives smaller than this many bytes are copied (not zero-copy)"); DEFINE_string(urma_device, "", "The name of the URMA device to use. Empty means the first one."); +DEFINE_string(urma_tp_type, "auto", + "RM transport path: auto (prefer CTP, then RTP), ctp, or rtp"); DEFINE_int32(urma_max_sge, 0, "Max SGEs per WR. 0 means the device maximum."); DEFINE_int32(urma_bonding_mode, 0, @@ -123,6 +129,7 @@ static bool g_is_bonding_device = false; // Prefer the device capability table and retain priority 6 as a compatibility // fallback for CTP providers that do not report a priority. static uint8_t g_jetty_priority = 6; +static urma_tp_type_t g_tp_type = URMA_CTP; // urma_init/urma_uninit manage process-global liburma state. Only uninitialize // it when this helper performed the successful initialization; URMA_EEXIST // means another component owns that state. @@ -383,6 +390,12 @@ bool InitPool() { g_pool_base = nullptr; return false; } + LOG(INFO) << "URMA registered buffer pool: base=" << g_pool_base + << " size=" << g_pool_size + << " segment_va=" << g_pool_seg->seg.ubva.va + << " segment_len=" << g_pool_seg->seg.len + << " segment_access=" << g_pool_seg->seg.attr.bs.access + << " segment_token_id=" << g_pool_seg->seg.token_id; g_pool->in_use.assign(count, 0); for (size_t i = 0; i < count; ++i) { auto shard = i % kShardCount; @@ -412,6 +425,70 @@ urma_target_seg_t* GetPoolSegFor(void* buf) { return nullptr; } +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment) { + if (!sge || !segment || length == 0 || + length > std::numeric_limits::max() || + segment->urma_ctx != GetUrmaContext()) { + errno = ERDMAMEM; + return false; + } + const uint64_t base = segment->seg.ubva.va; + const uint64_t size = segment->seg.len; + if (size > std::numeric_limits::max() - base || address < base || + address - base > size || length > size - (address - base)) { + errno = ERDMAMEM; + return false; + } + *sge = urma_sge_t{address, static_cast(length), segment, nullptr}; + return true; +} + +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*)) { + if (!buffer || !sge || !lookup) { + errno = EINVAL; + return false; + } + const size_t length = buffer->size(); + const size_t capacity = GetUrmaRecvBlockSize(); + if (length == 0 || capacity < URMA_SEND_BUFFER_ALIGNMENT || + length > capacity - (URMA_SEND_BUFFER_ALIGNMENT - 1)) { + errno = EMSGSIZE; + return false; + } + butil::IOBuf copied; + void* data = nullptr; + size_t padding = 0; + { + butil::IOBufAsZeroCopyOutputStream stream( + &copied, capacity + sizeof(butil::IOBuf::Block)); + int available = 0; + if (!stream.Next(&data, &available)) { + errno = ENOMEM; + return false; + } + padding = + (URMA_SEND_BUFFER_ALIGNMENT - + reinterpret_cast(data) % URMA_SEND_BUFFER_ALIGNMENT) % + URMA_SEND_BUFFER_ALIGNMENT; + if (padding + length > static_cast(available)) { + errno = ENOMEM; + return false; + } + data = static_cast(data) + padding; + buffer->copy_to(data, length); + stream.BackUp(available - static_cast(padding + length)); + } + copied.pop_front(padding); + if (!InitUrmaLocalSge(sge, reinterpret_cast(data), length, + lookup(data))) { + return false; + } + buffer->swap(copied); + return true; +} + static void GlobalRelease() { g_urma_available.store(false, butil::memory_order_release); if (g_mem_alloc_orig) { @@ -450,6 +527,7 @@ static void GlobalRelease() { g_has_local_eid = false; g_is_bonding_device = false; g_jetty_priority = 6; + g_tp_type = URMA_CTP; if (g_owns_urma_init) { const urma_status_t status = urma_uninit(); if (status != URMA_SUCCESS) { @@ -525,13 +603,22 @@ static bool GlobalUrmaInitializeImpl() { g_device = nullptr; return false; } - const int ctp_priority = - FindUrmaPriorityForTpType(g_device_attr, URMA_CTP); - if (ctp_priority >= 0) { - g_jetty_priority = static_cast(ctp_priority); - } else { - LOG(WARNING) << "URMA device does not report a CTP priority; " - "falling back to compatibility priority " + if (!SelectUrmaTransportPath(g_device_attr, FLAGS_urma_tp_type, &g_tp_type, + &g_jetty_priority)) { + LOG(ERROR) << "URMA path selection failed: requested=" + << FLAGS_urma_tp_type << " device=" << device_name + << " ctp_priority=" + << FindUrmaPriorityForTpType(g_device_attr, URMA_CTP) + << " rtp_priority=" + << FindUrmaPriorityForTpType(g_device_attr, URMA_RTP); + urma_free_device_list(devices); + g_device = nullptr; + errno = EINVAL; + return false; + } + if (FindUrmaPriorityForTpType(g_device_attr, g_tp_type) < 0) { + LOG(WARNING) << "URMA device does not report path capabilities; " + "using compatibility priority " << static_cast(g_jetty_priority); } uint32_t eid_cnt = 0; @@ -628,7 +715,11 @@ static bool GlobalUrmaInitializeImpl() { // resources on exit; GlobalRelease remains available for init rollback. LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device - << " max_sge=" << g_max_sge + << " tp_type=" << (g_tp_type == URMA_CTP ? "ctp" : "rtp") + << " priority=" << static_cast(g_jetty_priority) + << " max_sge=" << g_max_sge << " default_sl=" + << GetUrmaServiceLevelForPriority(g_jetty_priority) + << " copy_send=" << FLAGS_urma_copy_send << " buffer_size=" << g_pool_buffer_size << " buffer_count=" << g_pool->buffer_count(); return true; @@ -678,15 +769,146 @@ bool IsUrmaBondingDevice() { return g_is_bonding_device; } int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type) { const union urma_tp_type_en expected = TpTypeCapability(tp_type); + if (expected.value == 0) { + return -1; + } for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { - if (attr.dev_cap.priority_info[priority].tp_type.value == - expected.value) { + if ((attr.dev_cap.priority_info[priority].tp_type.value & + expected.value) != 0) { return priority; } } return -1; } +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority) { + if (!tp_type || !priority || + (requested != "auto" && requested != "ctp" && requested != "rtp")) { + return false; + } + for (auto candidate : {URMA_CTP, URMA_RTP}) { + if ((requested == "ctp" && candidate != URMA_CTP) || + (requested == "rtp" && candidate != URMA_RTP)) { + continue; + } + const int match = FindUrmaPriorityForTpType(attr, candidate); + if (match >= 0) { + *tp_type = candidate; + *priority = static_cast(match); + return true; + } + } + for (int i = 0; i <= URMA_MAX_PRIORITY; ++i) { + if (attr.dev_cap.priority_info[i].tp_type.value != 0) { + return false; + } + } + *tp_type = requested == "rtp" ? URMA_RTP : URMA_CTP; + *priority = *tp_type == URMA_CTP ? 6 : 0; + return true; +} +urma_tp_type_t GetUrmaTpType() { return g_tp_type; } uint8_t GetUrmaJettyPriority() { return g_jetty_priority; } +bool IsUrmaPrioritySupported(const urma_device_attr_t& attr, int priority, + urma_tp_type_t tp_type) { + return priority >= 0 && priority <= URMA_MAX_PRIORITY && + (tp_type == URMA_CTP || tp_type == URMA_RTP) && + (attr.dev_cap.priority_info[priority].tp_type.value & + TpTypeCapability(tp_type).value) != 0; +} + +static std::string DescribeUrmaSlMap(const urma_device_attr_t& attr, + urma_tp_type_t tp_type) { + std::ostringstream os; + os << '['; + bool first = true; + for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { + if (IsUrmaPrioritySupported(attr, priority, tp_type)) { + if (!first) { + os << ','; + } + first = false; + os << priority << ':' << attr.dev_cap.priority_info[priority].SL; + } + } + os << ']'; + return os.str(); +} + +bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, + const UrmaQosOptions& qos, urma_tp_type_t tp_type, + uint8_t default_priority, uint8_t* priority) { + if (!qos.IsValid() || !priority || + (tp_type != URMA_CTP && tp_type != URMA_RTP)) { + errno = EINVAL; + return false; + } + if (qos.priority == -1 && qos.service_level == -1) { + *priority = default_priority; + return true; + } + if (qos.priority == -1) { + for (int p = 0; p <= URMA_MAX_PRIORITY; ++p) { + if (IsUrmaPrioritySupported(attr, p, tp_type) && + attr.dev_cap.priority_info[p].SL == + static_cast(qos.service_level)) { + *priority = static_cast(p); + return true; + } + } + LOG(ERROR) << "URMA SL " << qos.service_level << " has no " + << (tp_type == URMA_CTP ? "CTP" : "RTP") + << " priority; supported priority:SL=" + << DescribeUrmaSlMap(attr, tp_type); + } else if (!IsUrmaPrioritySupported(attr, qos.priority, tp_type)) { + LOG(ERROR) << "URMA priority " << qos.priority << " does not advertise " + << (tp_type == URMA_CTP ? "CTP" : "RTP") + << " on the selected device; requested SL=" + << qos.service_level << "; supported priority:SL=" + << DescribeUrmaSlMap(attr, tp_type); + } else if (qos.service_level != -1 && + attr.dev_cap.priority_info[qos.priority].SL != + static_cast(qos.service_level)) { + LOG(ERROR) << "URMA priority " << qos.priority << " maps to SL " + << attr.dev_cap.priority_info[qos.priority].SL + << ", requested SL=" << qos.service_level + << "; supported priority:SL=" + << DescribeUrmaSlMap(attr, tp_type); + } else { + *priority = static_cast(qos.priority); + return true; + } + errno = EINVAL; + return false; +} + +bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { + if (!priority) { + errno = EINVAL; + return false; + } + if (qos.IsValid() && qos.priority == -1 && qos.service_level == -1) { + *priority = GetUrmaJettyPriority(); + return true; + } + if (!IsUrmaAvailable() || !g_context) { + LOG(ERROR) << "URMA QoS requires an initialized device: priority=" + << qos.priority << " SL=" << qos.service_level; + errno = EINVAL; + return false; + } + return ResolveUrmaJettyPriority(g_device_attr, qos, GetUrmaTpType(), + GetUrmaJettyPriority(), priority); +} + +int64_t GetUrmaServiceLevelForPriority(int priority) { + if (!g_context || + !IsUrmaPrioritySupported(g_device_attr, priority, GetUrmaTpType())) { + return -1; + } + return g_device_attr.dev_cap.priority_info[priority].SL; +} int GetUrmaMaxSge() { return g_max_sge; } size_t GetUrmaRecvBlockSize() { return g_recv_block_size; } diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 871f1a6354..c5284cb651 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -26,9 +26,14 @@ #include "bthread/types.h" #include "butil/atomicops.h" +namespace butil { +class IOBuf; +} + #if BRPC_WITH_URMA #include "urma_api.h" +#include "brpc/urma/urma_qos.h" #include "urma_types.h" namespace brpc { @@ -75,14 +80,43 @@ const urma_eid_t* GetUrmaLocalEid(); // Return true when the selected URMA device is a bonding provider device. bool IsUrmaBondingDevice(); -// Find the priority whose advertised transport-path capability exactly -// matches @tp_type. Returns -1 when the device does not report one. +// Find the first priority advertising @tp_type, including combined capabilities. +// Returns -1 when the device does not report one. int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type); -// Return the priority selected for the CTP jettys created by brpc. +// Select an RM path from "auto", "ctp", or "rtp". Auto prefers CTP, then RTP. +// Providers without a capability table retain CTP/6 or explicit RTP/0. +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority); + +// Return the process-wide path and priority selected at initialization. +urma_tp_type_t GetUrmaTpType(); uint8_t GetUrmaJettyPriority(); +// Check whether a priority advertises the selected RM path. +bool IsUrmaPrioritySupported(const urma_device_attr_t& attr, int priority, + urma_tp_type_t tp_type); +// Resolve a requested priority/SL pair against a device capability snapshot. +// Fully automatic options retain default_priority, including legacy fallback. +bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, + const UrmaQosOptions& qos, urma_tp_type_t tp_type, + uint8_t default_priority, uint8_t* priority); +// Resolve automatic selection or validate against the initialized device. +bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority); +// Return the advertised SL on the selected path, or -1 when no initialized mapping is available. +int64_t GetUrmaServiceLevelForPriority(int priority); + +// Initialize a local SGE whose complete byte range belongs to this context. +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment); + +constexpr size_t URMA_SEND_BUFFER_ALIGNMENT = 64; +// Copy host payloads into one aligned block and transfer its ownership to buffer. +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*) = GetPoolSegFor); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/src/brpc/urma/urma_qos.h b/src/brpc/urma/urma_qos.h new file mode 100644 index 0000000000..56f5c43d28 --- /dev/null +++ b/src/brpc/urma/urma_qos.h @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_QOS_H +#define BRPC_URMA_QOS_H + +namespace brpc { + +// Outbound RM QoS, resolved through the selected device's priority/SL map. +struct UrmaQosOptions { + // -1 selects from SL or the device default; explicit range [0, 15]. + int priority = -1; + // -1 leaves SL unspecified; explicit range [0, 255]. + int service_level = -1; + + bool IsValid() const { + return priority >= -1 && priority <= 15 && service_level >= -1 && + service_level <= 255; + } +}; + +} // namespace brpc + +#endif // BRPC_URMA_QOS_H diff --git a/src/brpc/urma/urma_recovery_policy.cpp b/src/brpc/urma/urma_recovery_policy.cpp new file mode 100644 index 0000000000..73bb6eda3a --- /dev/null +++ b/src/brpc/urma/urma_recovery_policy.cpp @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/time.h" +#include "bvar/passive_status.h" + +namespace brpc { +namespace urma { + +const char* UrmaFailureReasonName(UrmaFailureReason reason) { + switch (reason) { + case UrmaFailureReason::NONE: + return "none"; + case UrmaFailureReason::RETRY_EXHAUSTED: + return "retry_exhausted"; + case UrmaFailureReason::RNR: + return "rnr"; + case UrmaFailureReason::QP_FATAL: + return "qp_fatal"; + case UrmaFailureReason::CQ_ERROR: + return "cq_error"; + case UrmaFailureReason::CQ_POLL: + return "cq_poll"; + case UrmaFailureReason::CQ_EVENT: + return "cq_event"; + case UrmaFailureReason::CQ_NOTIFY: + return "cq_notify"; + case UrmaFailureReason::SEND_POST: + return "send_post"; + case UrmaFailureReason::ACCESS_ERROR: + return "access_error"; + case UrmaFailureReason::TRANSPORT_ERROR: + return "transport_error"; + case UrmaFailureReason::RECV_POST: + return "recv_post"; + default: + return "unknown"; + } +} + +UrmaRecoveryPolicy::UrmaRecoveryPolicy(Clock clock, size_t max_peers) + : _clock(clock) + , _max_peers(max_peers) + , _mode(UrmaRecoveryMode::LEGACY) + , _epoch(1) + , _cooldown_us(5000000) {} + +bool UrmaRecoveryPolicy::SetMode(UrmaRecoveryMode mode) { + if (mode != UrmaRecoveryMode::LEGACY && mode != UrmaRecoveryMode::AUTO && + mode != UrmaRecoveryMode::TCP_ONLY) { + return false; + } + std::lock_guard lock(_mutex); + if (_mode != mode) { + _mode = mode; + ++_epoch; + _peers.clear(); + } + return true; +} + +UrmaRecoveryMode UrmaRecoveryPolicy::GetMode() { + std::lock_guard lock(_mutex); + return _mode; +} + +bool UrmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { + if (cooldown_ms <= 0 || cooldown_ms > INT32_MAX) { + return false; + } + std::lock_guard lock(_mutex); + _cooldown_us = cooldown_ms * 1000; + return true; +} + +bool UrmaRecoveryPolicy::AllowNewConnection(const std::string& peer, + uint64_t* epoch) { + std::lock_guard lock(_mutex); + if (epoch != nullptr) { + *epoch = _epoch; + } + if (_mode == UrmaRecoveryMode::LEGACY) { + return true; + } + if (_mode == UrmaRecoveryMode::TCP_ONLY) { + ++_status.tcp_only_admissions; + return false; + } + const int64_t now = _clock(); + auto it = _peers.find(peer); + if (it == _peers.end()) { + return true; + } + if (now < it->second) { + ++_status.peer_cooldown_admissions; + return false; + } + _peers.erase(it); + return true; +} + +void UrmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, + uint64_t epoch, + UrmaFailureReason reason) { + std::lock_guard lock(_mutex); + if (epoch != _epoch || reason <= UrmaFailureReason::NONE || + reason >= UrmaFailureReason::COUNT) { + return; + } + ++_status.failures[static_cast(reason)]; + if (_mode != UrmaRecoveryMode::AUTO || peer.empty() || _max_peers == 0 || + reason == UrmaFailureReason::ACCESS_ERROR) { + return; + } + const int64_t now = _clock(); + for (auto it = _peers.begin(); it != _peers.end();) { + if (it->second <= now) { + it = _peers.erase(it); + } else { + ++it; + } + } + if (_peers.count(peer) == 0 && _peers.size() == _max_peers) { + auto oldest = _peers.begin(); + for (auto it = _peers.begin(); it != _peers.end(); ++it) { + if (it->second < oldest->second) { + oldest = it; + } + } + _peers.erase(oldest); + ++_status.cache_evictions; + } + _peers[peer] = now + _cooldown_us; +} + +UrmaRecoveryStatus UrmaRecoveryPolicy::GetStatus() { + std::lock_guard lock(_mutex); + UrmaRecoveryStatus result = _status; + result.mode = _mode; + result.epoch = _epoch; + const int64_t now = _clock(); + for (const auto& peer : _peers) { + result.cooled_peers += peer.second > now; + } + return result; +} + +void UrmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { + std::lock_guard lock(_mutex); + if (epoch == _epoch) { + ++_status.negotiated_connections; + } +} + +static void DescribeRecovery(std::ostream& os, void* arg) { + const auto s = static_cast(arg)->GetStatus(); + os << "{\"mode\":\"" + << (s.mode == UrmaRecoveryMode::AUTO ? "auto" + : s.mode == UrmaRecoveryMode::TCP_ONLY ? "tcp_only" + : "legacy") + << "\",\"epoch\":" << s.epoch << ",\"cooled_peers\":" << s.cooled_peers + << ",\"tcp_only_admissions\":" << s.tcp_only_admissions + << ",\"peer_cooldown_admissions\":" << s.peer_cooldown_admissions + << ",\"cache_evictions\":" << s.cache_evictions + << ",\"negotiated_connections\":" << s.negotiated_connections + << ",\"failures\":{"; + for (size_t i = 1; i < static_cast(UrmaFailureReason::COUNT); ++i) { + if (i > 1) { + os << ','; + } + os << '"' << UrmaFailureReasonName(static_cast(i)) + << "\":" << s.failures[i]; + } + os << "}}"; +} + +struct RecoveryMetrics { + RecoveryMetrics(UrmaRecoveryPolicy* policy, const char* status_name, + const char* fallback_name) + : status(status_name, DescribeRecovery, policy) + , fallbacks( + fallback_name, + [](void* arg) { + const auto s = + static_cast(arg)->GetStatus(); + return s.tcp_only_admissions + s.peer_cooldown_admissions; + }, + policy) {} + bvar::PassiveStatus status; + bvar::PassiveStatus fallbacks; +}; + +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy() { + static UrmaRecoveryPolicy* policy = + new UrmaRecoveryPolicy(butil::monotonic_time_us); + static RecoveryMetrics* metrics = + new RecoveryMetrics(policy, "urma_recovery_status", + "urma_recovery_tcp_fallback_admissions"); + (void)metrics; + return *policy; +} + +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode) { + return GlobalUrmaRecoveryPolicy().SetMode(mode); +} + +UrmaRecoveryMode GetUrmaRecoveryMode() { + return GlobalUrmaRecoveryPolicy().GetMode(); +} + +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms) { + return GlobalUrmaRecoveryPolicy().SetCooldownMs(cooldown_ms); +} + +UrmaRecoveryStatus GetUrmaRecoveryStatus() { + return GlobalUrmaRecoveryPolicy().GetStatus(); +} + +} // namespace urma +} // namespace brpc diff --git a/src/brpc/urma/urma_recovery_policy.h b/src/brpc/urma/urma_recovery_policy.h new file mode 100644 index 0000000000..101b629de8 --- /dev/null +++ b/src/brpc/urma/urma_recovery_policy.h @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_RECOVERY_POLICY_H +#define BRPC_URMA_RECOVERY_POLICY_H + +#include +#include +#include +#include +#include + +namespace brpc { +namespace urma { + +enum class UrmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; + +enum class UrmaFailureReason { + NONE, + RETRY_EXHAUSTED, + RNR, + QP_FATAL, + CQ_ERROR, + CQ_POLL, + CQ_EVENT, + CQ_NOTIFY, + SEND_POST, + ACCESS_ERROR, + TRANSPORT_ERROR, + RECV_POST, + COUNT +}; + +const char* UrmaFailureReasonName(UrmaFailureReason reason); + +struct UrmaRecoveryStatus { + UrmaRecoveryMode mode = UrmaRecoveryMode::LEGACY; + uint64_t epoch = 0; + size_t cooled_peers = 0; + uint64_t failures[static_cast(UrmaFailureReason::COUNT)] = {}; + uint64_t tcp_only_admissions = 0; + uint64_t peer_cooldown_admissions = 0; + uint64_t cache_evictions = 0; + uint64_t negotiated_connections = 0; +}; + +// Admission policy for new URMA connections. +// An empty peer selects server admission, which only checks the mode. +class UrmaRecoveryPolicy { +public: + typedef int64_t (*Clock)(); + explicit UrmaRecoveryPolicy(Clock clock, size_t max_peers = 1024); + bool SetMode(UrmaRecoveryMode mode); + UrmaRecoveryMode GetMode(); + // Positive milliseconds, at most INT32_MAX. Applies to future failures. + bool SetCooldownMs(int64_t cooldown_ms); + bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); + void ReportPeerFailure( + const std::string& peer, uint64_t epoch, + UrmaFailureReason reason = UrmaFailureReason::RETRY_EXHAUSTED); + UrmaRecoveryStatus GetStatus(); + void RecordNegotiatedConnection(uint64_t epoch); + +private: + std::mutex _mutex; + const Clock _clock; + const size_t _max_peers; + UrmaRecoveryMode _mode; + uint64_t _epoch; + int64_t _cooldown_us; + std::map _peers; + UrmaRecoveryStatus _status; +}; + +// Process-lifetime policy, shared by client and server handshake admission. +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy(); +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode); +UrmaRecoveryMode GetUrmaRecoveryMode(); +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms); +UrmaRecoveryStatus GetUrmaRecoveryStatus(); + +} // namespace urma +} // namespace brpc + +#endif // BRPC_URMA_RECOVERY_POLICY_H diff --git a/src/brpc/urma_transport.cpp b/src/brpc/urma_transport.cpp index c5cc163ffa..9d88c5e1d0 100644 --- a/src/brpc/urma_transport.cpp +++ b/src/brpc/urma_transport.cpp @@ -80,7 +80,10 @@ int UrmaTransport::Reset(int32_t /*expected_nref*/) { } std::shared_ptr UrmaTransport::Connect() { - if (_default_connect == nullptr) { + // Child sockets inherit AppConnect options. The built-in handshake + // stores a pending callback and therefore belongs to one socket only. + if (!_default_connect || + dynamic_cast(_default_connect.get())) { return std::make_shared(); } return _default_connect; @@ -197,6 +200,12 @@ int UrmaTransport::ContextInitOrDie(bool server_or_not, const void* options) { return -1; } urma::GlobalUrmaInitializeOrDie(); + uint8_t priority; + if (!urma::ResolveUrmaJettyPriority( + static_cast(options)->urma_qos, + &priority)) { + return -1; + } if (!urma::InitPollingModeWithTag( static_cast(options)->bthread_tag)) { return -1; @@ -207,6 +216,12 @@ int UrmaTransport::ContextInitOrDie(bool server_or_not, const void* options) { return -1; } urma::GlobalUrmaInitializeOrDie(); + uint8_t priority; + if (!urma::ResolveUrmaJettyPriority( + static_cast(options)->urma_qos, + &priority)) { + return -1; + } if (!urma::InitPollingModeWithTag(bthread_self_tag())) { return -1; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8800daeb40..c83466f23d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -319,6 +319,17 @@ if(WITH_FLATBUFFERS) endif() # brpc tests +if(WITH_URMA AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_library(brpc_urma_send_failure_injector MODULE + ../example/urma_performance/send_failure_injector.c) + set_target_properties(brpc_urma_send_failure_injector PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON) + target_include_directories(brpc_urma_send_failure_injector PRIVATE + $) + target_link_libraries(brpc_urma_send_failure_injector PRIVATE + ${CMAKE_DL_LIBS} Threads::Threads) +endif() + file(GLOB BRPC_UNITTESTS "brpc_*_unittest.cpp") list(REMOVE_ITEM BRPC_UNITTESTS "${CMAKE_CURRENT_SOURCE_DIR}/brpc_flatbuffers_unittest.cpp") @@ -330,7 +341,44 @@ foreach(BRPC_UT ${BRPC_UNITTESTS}) gtest_main brpc_test_config ${GPERFTOOLS_LIBRARIES}) - add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE}) + if(BRPC_UT_WE STREQUAL "brpc_urma_remote_recovery_unittest" AND + TARGET brpc_urma_send_failure_injector) + add_dependencies(${BRPC_UT_WE} brpc_urma_send_failure_injector) + foreach(version 2 3) + foreach(path ctp rtp) + foreach(polling false true) + set(remote_test "brpc_urma_remote_recovery_${path}_v${version}_${polling}_unittest") + add_test(NAME ${remote_test} COMMAND ${BRPC_UT_WE} + --urma_tp_type=${path} + --urma_client_handshake_version=${version} + --urma_use_polling=${polling}) + set_tests_properties(${remote_test} PROPERTIES + ENVIRONMENT "LD_PRELOAD=$") + endforeach() + endforeach() + endforeach() + else() + add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE}) + endif() + if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_qos_unittest") + add_test(NAME brpc_urma_qos_rtp_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + add_test(NAME brpc_urma_qos_rtp_polling_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp --urma_use_polling=true) + endif() + if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_qos_churn_unittest") + add_test(NAME brpc_urma_qos_churn_rtp_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + add_test(NAME brpc_urma_qos_churn_rtp_polling_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp --urma_use_polling=true) + endif() + if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_transport_unittest") + add_test(NAME brpc_urma_transport_rtp_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + add_test(NAME brpc_urma_transport_rtp_stress_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp + --gtest_shuffle --gtest_random_seed=92 --gtest_repeat=20) + endif() endforeach() if(BUILD_FUZZ_TESTS) diff --git a/test/brpc_urma_copy_unittest.cpp b/test/brpc_urma_copy_unittest.cpp new file mode 100644 index 0000000000..fbb61d5a97 --- /dev/null +++ b/test/brpc_urma_copy_unittest.cpp @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_bool(urma_copy_send); +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); +DECLARE_int32(urma_max_sge); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class CopiedEchoService : public test::EchoService { + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +class UrmaCopiedRpcTest : public ::testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (!_mock) { + return; + } + brpc::urma::FLAGS_urma_copy_send = true; + brpc::urma::FLAGS_urma_buffer_count = 2048; + brpc::urma::FLAGS_urma_prepared_jetty_cnt = 0; + brpc::urma::FLAGS_urma_max_sge = 1; + brpc::urma::GlobalUrmaInitializeOrDie(); + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "Copied URMA RPC tests use the link-time mock"; + } + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + ASSERT_NE(nullptr, brpc::urma::GetUrmaContext()); + brpc::urma::FLAGS_urma_client_handshake_version = GetParam(); + } + static bool _mock; +}; +bool UrmaCopiedRpcTest::_mock = false; + +TEST_P(UrmaCopiedRpcTest, + copies_payload_and_attachment_across_wrs_and_credit_returns) { + CopiedEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "single"; + co.timeout_ms = 3000; + co.max_retry = 0; + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + // Exceed the send window to exercise receive-credit ACKs as well as data WRs. + for (int round = 0; round < 40; ++round) { + const size_t size = round == 0 ? 0 : 20000; + const std::string payload(size, 'v'); + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message(payload); + cntl.request_attachment().append(payload); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(payload, response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + } + EXPECT_EQ(before.negotiated_connections + 2, + brpc::urma::GetUrmaRecoveryStatus().negotiated_connections); + server.Stop(0); + server.Join(); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaCopiedRpcTest, + ::testing::Values(2, 3)); + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_mock_completion_unittest.cpp b/test/brpc_urma_mock_completion_unittest.cpp new file mode 100644 index 0000000000..76cd2c2912 --- /dev/null +++ b/test/brpc_urma_mock_completion_unittest.cpp @@ -0,0 +1,260 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "urma_api.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_helper.h" + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static ssize_t Send(Socket* socket, urma_jetty_t* jetty, + urma_target_jetty_t* target, butil::IOBuf** buffers) { + UrmaEndpoint endpoint(socket); + UrmaResource resource; + resource.jetty = jetty; + resource.remote_jetty = target; + endpoint._resource = &resource; + endpoint._sbuf.resize(2); + endpoint._sq_window_size.store(4, butil::memory_order_relaxed); + endpoint._remote_rq_window_size.store(4, butil::memory_order_relaxed); + endpoint._remote_recv_block_size = 4; + const ssize_t sent = endpoint.CutFromIOBufList(buffers, 2); + endpoint._resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return sent; + } +}; + +} // namespace urma +} // namespace brpc + +namespace { + +class UrmaMockCompletionTest : public testing::Test { + protected: + void SetUp() override { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + _owns_init = status == URMA_SUCCESS; + int count = 0; + auto** devices = urma_get_device_list(&count); + const bool mock = + devices && count > 0 && + std::strcmp(devices[0]->name, "mock_urma_device") == 0; + if (mock) { + _context = urma_create_context(devices[0], 0); + } + urma_free_device_list(devices); + if (!mock) { + GTEST_SKIP() << "Completion ordering test uses the link-time mock"; + } + ASSERT_NE(nullptr, _context); + urma_jfc_cfg_t cq{}; + cq.depth = 8; + _send_cq = urma_create_jfc(_context, &cq); + _recv_cq = urma_create_jfc(_context, &cq); + ASSERT_NE(nullptr, _send_cq); + ASSERT_NE(nullptr, _recv_cq); + urma_jfr_cfg_t rq{}; + rq.depth = 8; + rq.jfc = _recv_cq; + _recv_queue = urma_create_jfr(_context, &rq); + ASSERT_NE(nullptr, _recv_queue); + urma_jetty_cfg_t cfg{}; + cfg.jfs_cfg.jfc = _send_cq; + cfg.shared.jfr = _recv_queue; + _sender = urma_create_jetty(_context, &cfg); + _receiver = urma_create_jetty(_context, &cfg); + ASSERT_NE(nullptr, _sender); + ASSERT_NE(nullptr, _receiver); + urma_rjetty_t remote{}; + remote.jetty_id = _receiver->jetty_id; + urma_token_t token{}; + _target = urma_import_jetty(_context, &remote, &token); + ASSERT_NE(nullptr, _target); + } + + void TearDown() override { + if (_target) urma_unimport_jetty(_target); + if (_receiver) urma_delete_jetty(_receiver); + if (_sender) urma_delete_jetty(_sender); + if (_recv_queue) urma_delete_jfr(_recv_queue); + if (_recv_cq) urma_delete_jfc(_recv_cq); + if (_send_cq) urma_delete_jfc(_send_cq); + if (_context) urma_delete_context(_context); + if (_owns_init) urma_uninit(); + } + + bool _owns_init = false; + urma_context_t* _context = nullptr; + urma_jfc_t* _send_cq = nullptr; + urma_jfc_t* _recv_cq = nullptr; + urma_jfr_t* _recv_queue = nullptr; + urma_jetty_t* _sender = nullptr; + urma_jetty_t* _receiver = nullptr; + urma_target_jetty_t* _target = nullptr; +}; + +TEST_F(UrmaMockCompletionTest, undeliverable_send_has_no_success_completion) { + urma_jfs_wr_t send{}; + send.opcode = URMA_OPC_SEND; + send.flag.bs.complete_enable = 1; + send.tjetty = _target; + urma_jfs_wr_t* bad = nullptr; + urma_cr_t cr{}; + + EXPECT_EQ(URMA_EAGAIN, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); + + ASSERT_EQ(URMA_SUCCESS, urma_delete_jfc(_recv_cq)); + _recv_cq = nullptr; + bad = nullptr; + EXPECT_NE(URMA_SUCCESS, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); + + ASSERT_EQ(URMA_SUCCESS, urma_delete_jetty(_receiver)); + _receiver = nullptr; + bad = nullptr; + EXPECT_NE(URMA_SUCCESS, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); +} + +TEST_F(UrmaMockCompletionTest, provider_backpressure_keeps_unsent_suffix) { + brpc::SocketId sid; + ASSERT_EQ(0, brpc::Socket::Create(brpc::SocketOptions(), &sid)); + brpc::SocketUniquePtr socket; + ASSERT_EQ(0, brpc::Socket::Address(sid, &socket)); + + char payload[] = "abcdefghijk"; + urma_target_seg_t segment{}; + segment.urma_ctx = brpc::urma::GetUrmaContext(); + segment.seg.ubva.va = reinterpret_cast(payload); + segment.seg.len = sizeof(payload) - 1; + const auto meta = reinterpret_cast(&segment); + butil::IOBuf head; + butil::IOBuf tail; + ASSERT_EQ(0, head.append_user_data_with_meta(payload, 6, [](void*) {}, meta)); + ASSERT_EQ(0, tail.append_user_data_with_meta(payload + 6, 5, + [](void*) {}, meta)); + butil::IOBuf* buffers[] = {&head, &tail}; + + char received[4]{}; + urma_sge_t sge{}; + sge.addr = reinterpret_cast(received); + sge.len = sizeof(received); + urma_jfr_wr_t recv{}; + recv.src.sge = &sge; + recv.src.num_sge = 1; + urma_jfr_wr_t* bad = nullptr; + ASSERT_EQ(URMA_SUCCESS, urma_post_jfr_wr(_recv_queue, &recv, &bad)); + + EXPECT_EQ(4, brpc::urma::UrmaRecoveryTestPeer::Send( + socket.get(), _sender, _target, buffers)); + EXPECT_EQ("abcd", std::string(received, sizeof(received))); + EXPECT_EQ("ef", head.to_string()); + EXPECT_EQ("ghijk", tail.to_string()); + EXPECT_EQ(-1, brpc::urma::UrmaRecoveryTestPeer::Send( + socket.get(), _sender, _target, buffers)); + EXPECT_EQ(EAGAIN, errno); + EXPECT_EQ("ef", head.to_string()); + EXPECT_EQ("ghijk", tail.to_string()); + socket->SetFailed(); +} + +TEST_F(UrmaMockCompletionTest, send_completion_follows_payload_delivery) { + // Keep the copy large enough for a concurrent CQ consumer to run while + // the mock is still using the source buffer. + const std::string payload(16 * 1024 * 1024, 'p'); + std::string received(payload.size(), '\0'); + urma_sge_t recv_sge{}; + recv_sge.addr = reinterpret_cast(&received[0]); + recv_sge.len = received.size(); + urma_jfr_wr_t recv_wr{}; + recv_wr.src.sge = &recv_sge; + recv_wr.src.num_sge = 1; + recv_wr.user_ctx = 2; + urma_jfr_wr_t* bad_recv = nullptr; + ASSERT_EQ(URMA_SUCCESS, urma_post_jfr_wr(_recv_queue, &recv_wr, &bad_recv)); + + urma_sge_t send_sge{}; + send_sge.addr = reinterpret_cast(payload.data()); + send_sge.len = payload.size(); + urma_jfs_wr_t send_wr{}; + send_wr.opcode = URMA_OPC_SEND; + send_wr.flag.bs.complete_enable = 1; + send_wr.tjetty = _target; + send_wr.send.src.sge = &send_sge; + send_wr.send.src.num_sge = 1; + send_wr.user_ctx = 1; + std::atomic started{false}; + std::atomic finished{false}; + urma_status_t posted = URMA_EINVAL; + std::thread writer([&] { + while (!started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + urma_jfs_wr_t* bad_send = nullptr; + posted = urma_post_jetty_send_wr(_sender, &send_wr, &bad_send); + finished.store(true, std::memory_order_release); + }); + started.store(true, std::memory_order_release); + urma_cr_t send_cr{}; + int send_count = 0; + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (send_count == 0 && std::chrono::steady_clock::now() < deadline) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + if (finished.load(std::memory_order_acquire) && send_count == 0) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + break; + } + } + urma_cr_t recv_cr{}; + const int recv_count = urma_poll_jfc(_recv_cq, 1, &recv_cr); + // Join before assertions so a failed check still releases the worker. + writer.join(); + ASSERT_EQ(URMA_SUCCESS, posted); + ASSERT_EQ(1, send_count); + EXPECT_EQ(URMA_CR_SUCCESS, send_cr.status); + EXPECT_EQ(1u, send_cr.user_ctx); + ASSERT_EQ(1, recv_count) + << "Mock SEND completed before delivering its source payload"; + EXPECT_EQ(URMA_CR_SUCCESS, recv_cr.status); + EXPECT_EQ(2u, recv_cr.user_ctx); + EXPECT_EQ(payload.size(), recv_cr.completion_len); + EXPECT_EQ(payload, received); +} + +} // namespace +#endif // BRPC_WITH_URMA diff --git a/test/brpc_urma_qos_churn_unittest.cpp b/test/brpc_urma_qos_churn_unittest.cpp new file mode 100644 index 0000000000..2bc6d96ee9 --- /dev/null +++ b/test/brpc_urma_qos_churn_unittest.cpp @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "bthread/bthread.h" +#include "butil/time.h" +#include "brpc/acceptor.h" +#include "brpc/channel.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "brpc/urma_transport.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); +DECLARE_int32(urma_client_handshake_version); +extern bool g_skip_urma_init; + +class UrmaQosTestPeer { +public: + static int Priority(UrmaEndpoint* endpoint) { + return endpoint->_resource && endpoint->_resource->jetty + ? endpoint->_resource->jetty->jetty_cfg.jfs_cfg.priority + : -1; + } + static ssize_t FailCompletion(UrmaEndpoint* endpoint) { + urma_cr_t cr{}; + cr.status = URMA_CR_RNR_RETRY_CNT_EXC_ERR; + return endpoint->HandleCompletion(cr); + } +}; +} // namespace urma +} // namespace brpc + +namespace { +using namespace brpc; + +bool WaitUntil(const std::function& ready) { + const int64_t deadline = butil::monotonic_time_us() + 5000000; + while (!ready()) { + if (butil::monotonic_time_us() >= deadline) { + return false; + } + bthread_usleep(1000); + } + return true; +} + +class QosEcho : public test::EchoService { + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { + ClosureGuard guard(done); + auto* controller = static_cast(base); + response->set_message(request->message()); + controller->response_attachment().append( + controller->request_attachment()); + } +}; + +class UrmaQosChurnTest : public testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + std::strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (_mock) { + urma::g_skip_urma_init = false; + urma::FLAGS_urma_prepared_jetty_cnt = 2; + urma::GlobalUrmaInitializeOrDie(); + } + } + + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "URMA churn tests use the link-time mock"; + } + ASSERT_TRUE(urma::IsUrmaAvailable()); + _saved_mode = urma::GetUrmaRecoveryMode(); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + ASSERT_TRUE(urma::SetUrmaRecoveryCooldownMs(1000)); + urma::FLAGS_urma_client_handshake_version = GetParam(); + ASSERT_EQ(0, _server.AddService(&_service, SERVER_DOESNT_OWN_SERVICE)); + ServerOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.service_level = 2; + ASSERT_EQ(0, _server.Start("127.0.0.1:0", &options)); + _options.socket_mode = SOCKET_MODE_URMA; + _options.urma_qos.service_level = 4; + _options.timeout_ms = 2000; + _options.connect_timeout_ms = 2000; + _options.max_retry = 0; + _options.connection_type = "single"; + } + + void TearDown() override { + if (_mock) { + _server.Stop(0); + _server.Join(); + EXPECT_TRUE(WaitUntil([this] { + for (auto id : _client_ids) { + SocketUniquePtr socket; + if (Socket::AddressFailedAsWell(id, &socket) >= 0) { + return false; + } + } + return true; + })) << "Client sockets retained after test completion"; + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(_saved_mode); + urma::SetUrmaRecoveryCooldownMs(5000); + } + } + + void Echo(Channel* channel, int sequence) { + Controller controller; + test::EchoRequest request; + test::EchoResponse response; + request.set_message("QoS lifecycle " + std::to_string(sequence)); + std::string payload(65536, '\0'); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast((i + sequence) % 251); + } + controller.request_attachment().append(payload); + test::EchoService_Stub(channel).Echo(&controller, &request, &response, + nullptr); + ASSERT_FALSE(controller.Failed()) << controller.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(payload, controller.response_attachment().to_string()); + } + + void CheckConnection(Channel* channel, bool use_urma) { + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(channel->_server_id, &socket)); + EXPECT_EQ(4, socket->urma_qos().service_level); + EXPECT_EQ(-1, socket->urma_qos().priority); + auto* transport = static_cast(socket->_transport.get()); + EXPECT_EQ(use_urma ? UrmaTransport::URMA_ON : UrmaTransport::URMA_OFF, + transport->_urma_state.load()); + EXPECT_EQ(use_urma ? 10 : -1, + urma::UrmaQosTestPeer::Priority(transport->GetUrmaEp())); + std::vector accepted; + _server._am->ListConnections(&accepted); + bool found = false; + for (auto id : accepted) { + SocketUniquePtr peer; + if (Socket::Address(id, &peer) == 0 && + peer->remote_side() == socket->local_side()) { + found = true; + EXPECT_EQ(2, peer->urma_qos().service_level); + auto* peer_transport = + static_cast(peer->_transport.get()); + const int expected_priority = + use_urma ? (urma::GetUrmaTpType() == URMA_RTP ? 0 : 6) : -1; + EXPECT_EQ(expected_priority, urma::UrmaQosTestPeer::Priority( + peer_transport->GetUrmaEp())); + } + } + EXPECT_TRUE(found); + } + + static bool _mock; + GFLAGS_NAMESPACE::FlagSaver _flags; + urma::UrmaRecoveryMode _saved_mode = urma::UrmaRecoveryMode::LEGACY; + QosEcho _service; + Server _server; + ChannelOptions _options; + std::vector _client_ids; +}; +bool UrmaQosChurnTest::_mock = false; + +TEST_P(UrmaQosChurnTest, short_connections_release_resources) { + _options.connection_type = "short"; + Channel channel; + ASSERT_EQ(0, channel.Init(_server.listen_address(), &_options)); + _client_ids.push_back(channel._server_id); + for (int i = 0; i < 64; ++i) { + ASSERT_NO_FATAL_FAILURE(Echo(&channel, i)); + ASSERT_TRUE(WaitUntil([this] { + return _server._am->ConnectionCount() == 0; + })) << "Accepted short connection retained after RPC " + << i; + } +} + +TEST_P(UrmaQosChurnTest, cooldown_and_fresh_connections_preserve_business_qos) { + Channel original; + ASSERT_EQ(0, original.Init(_server.listen_address(), &_options)); + _client_ids.push_back(original._server_id); + ASSERT_NO_FATAL_FAILURE(Echo(&original, 1)); + ASSERT_NO_FATAL_FAILURE(CheckConnection(&original, true)); + const auto baseline = urma::GetUrmaRecoveryStatus(); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(original._server_id, &socket)); + auto* transport = static_cast(socket->_transport.get()); + EXPECT_EQ(-1, + urma::UrmaQosTestPeer::FailCompletion(transport->GetUrmaEp())); + socket->SetFailed(EIO, "Injected completion error"); + socket.reset(); + + _options.connection_group = "qos-cooldown"; + Channel tcp; + ASSERT_EQ(0, tcp.Init(_server.listen_address(), &_options)); + _client_ids.push_back(tcp._server_id); + ASSERT_NO_FATAL_FAILURE(Echo(&tcp, 2)); + ASSERT_NO_FATAL_FAILURE(CheckConnection(&tcp, false)); + EXPECT_GT(urma::GetUrmaRecoveryStatus().peer_cooldown_admissions, + baseline.peer_cooldown_admissions); + ASSERT_TRUE(WaitUntil( + [] { return urma::GetUrmaRecoveryStatus().cooled_peers == 0; })); + + _options.connection_group = "qos-recovered"; + Channel recovered; + ASSERT_EQ(0, recovered.Init(_server.listen_address(), &_options)); + _client_ids.push_back(recovered._server_id); + ASSERT_NO_FATAL_FAILURE(Echo(&recovered, 3)); + ASSERT_NO_FATAL_FAILURE(CheckConnection(&recovered, true)); + const auto after = urma::GetUrmaRecoveryStatus(); + const size_t reason = static_cast(urma::UrmaFailureReason::RNR); + EXPECT_EQ(baseline.failures[reason] + 1, after.failures[reason]); + EXPECT_GT(after.negotiated_connections, baseline.negotiated_connections); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaQosChurnTest, + testing::Values(2, 3)); +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); +#if BRPC_WITH_URMA + brpc::urma::FLAGS_urma_buffer_count = 4096; +#endif + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_qos_unittest.cpp b/test/brpc_urma_qos_unittest.cpp new file mode 100644 index 0000000000..3a8e89c851 --- /dev/null +++ b/test/brpc_urma_qos_unittest.cpp @@ -0,0 +1,799 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "butil/logging.h" +#include "butil/memory/scope_guard.h" +#include "brpc/global.h" +#include "brpc/acceptor.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/load_balancer.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_handshake.h" +#include "brpc/urma_transport.h" +#include "echo.pb.h" + +using namespace brpc; + +namespace brpc { +namespace urma { +DECLARE_int32(urma_client_handshake_version); +DECLARE_string(urma_tp_type); +extern bool g_skip_urma_init; +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); + +class UrmaQosTestPeer { +public: + static int Allocate(UrmaEndpoint* ep) { return ep->AllocateResources(); } + static int Priority(UrmaEndpoint* ep) { + return ep->_resource->jetty->jetty_cfg.jfs_cfg.priority; + } + static ssize_t Complete(UrmaEndpoint* ep, const urma_cr_t& cr) { + return ep->HandleCompletion(cr); + } + static int Import(UrmaEndpoint* ep, const ParsedHello& hello) { + return ep->ImportPeer(hello); + } + static ParsedHello Hello(UrmaEndpoint* ep) { + ParsedHello hello; + ep->MakeLocalParsedHello(&hello); + return hello; + } +}; + +} // namespace urma +} // namespace brpc + +TEST(UrmaHelperTest, explicit_priority_requires_ctp_capability) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 0, URMA_CTP)); + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 15, URMA_CTP)); + for (int priority : {-2, -1, 3, 4, 16}) { + EXPECT_FALSE(urma::IsUrmaPrioritySupported(attr, priority, URMA_CTP)); + } +} + +TEST(UrmaHelperTest, automatic_priority_accepts_combined_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + EXPECT_EQ(3, urma::FindUrmaPriorityForTpType(attr, URMA_RTP)); + EXPECT_EQ(10, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 10, URMA_CTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( + attr, static_cast(-1))); + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 0; + EXPECT_EQ(15, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 0; + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); +} + +TEST(UrmaHelperTest, service_level_selects_ctp_priority_from_mapping) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].SL = 6; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 7; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].SL = 6; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].SL = 6; + UrmaQosOptions qos; + uint8_t priority = 0; + qos.service_level = 6; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(10, priority); + qos.service_level = 7; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(6, priority); + qos.priority = 6; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(6, priority); + attr.dev_cap.priority_info[15].SL = 255; + qos.priority = -1; + qos.service_level = 255; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(15, priority); + qos.service_level = -1; + attr = urma_device_attr_t{}; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(6, priority); +} + +TEST(UrmaHelperTest, + service_level_rejects_unsupported_and_conflicting_mapping) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].SL = 3; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 2; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].SL = 4; + UrmaQosOptions qos; + uint8_t priority = 255; + for (int sl : {-2, 0, 3, 6, 255, 256}) { + qos.service_level = sl; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ(255, priority); + } + qos.priority = 6; + qos.service_level = 4; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ(255, priority); +#if !BRPC_WITH_GLOG + qos.priority = -1; + qos.service_level = 3; + logging::StringSink log; + logging::LogSink* old_sink = logging::SetLogSink(&log); + const bool resolved = + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority); + logging::SetLogSink(old_sink); + EXPECT_FALSE(resolved); + EXPECT_NE(std::string::npos, log.find("SL 3 has no CTP priority")) << log; + EXPECT_NE(std::string::npos, log.find("[6:2,10:4]")) << log; +#endif +} + +TEST(UrmaPathQosTest, automatic_and_explicit_paths_use_device_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 15; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_CTP, path); + EXPECT_EQ(6, priority); + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); +} + +TEST(UrmaPathQosTest, unsupported_paths_preserve_outputs) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const char* request : {"ctp", "utp", "", "typo"}) { + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, request, &path, &priority)); + EXPECT_EQ(URMA_UTP, path); + EXPECT_EQ(15, priority); + } + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, "rtp", nullptr, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, nullptr)); + attr.dev_cap.priority_info[0].tp_type.value = 0; + attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); +} + +TEST(UrmaPathQosTest, empty_capability_table_uses_compatibility_defaults) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const char* request : {"auto", "ctp", "rtp"}) { + ASSERT_TRUE( + urma::SelectUrmaTransportPath(attr, request, &path, &priority)); + const bool rtp = std::string(request) == "rtp"; + EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(rtp ? 0 : 6, priority); + UrmaQosOptions qos; + uint8_t resolved = 15; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, path, priority, + &resolved)); + EXPECT_EQ(priority, resolved); + qos.service_level = 2; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, path, priority, + &resolved)); + } +} + +TEST(UrmaPathQosTest, service_level_resolution_is_scoped_to_the_selected_path) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + attr.dev_cap.priority_info[4].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[4].SL = 4; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 2; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].SL = 4; + UrmaQosOptions qos; + qos.service_level = 2; + uint8_t priority = 15; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(0, priority); + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(6, priority); + qos.service_level = 4; + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(4, priority); + qos.priority = 10; + for (auto path : {URMA_RTP, URMA_CTP}) { + ASSERT_TRUE( + urma::ResolveUrmaJettyPriority(attr, qos, path, 0, &priority)); + EXPECT_EQ(10, priority); + } +} + +TEST(UrmaPathQosTest, rejects_cross_path_priorities_and_conflicting_sl) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 6; + UrmaQosOptions qos; + uint8_t priority = 15; + qos.service_level = 6; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + qos.priority = 6; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + qos.priority = 0; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(15, priority); + EXPECT_EQ(EINVAL, errno); +} + +TEST(UrmaPathQosTest, rejects_invalid_resolution_arguments) { + urma_device_attr_t attr{}; + UrmaQosOptions qos; + uint8_t priority = 15; + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, nullptr)); + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(qos, nullptr)); + for (auto path : {URMA_UTP, static_cast(255)}) { + EXPECT_FALSE( + urma::ResolveUrmaJettyPriority(attr, qos, path, 0, &priority)); + EXPECT_FALSE(urma::IsUrmaPrioritySupported(attr, 0, path)); + } + EXPECT_EQ(15, priority); + EXPECT_EQ(EINVAL, errno); +} + +class UrmaQosTest : public ::testing::Test { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (!_mock) { + return; + } + urma::g_skip_urma_init = false; + urma::FLAGS_urma_buffer_count = 1024; + urma::FLAGS_urma_prepared_jetty_cnt = 2; + urma::GlobalUrmaInitializeOrDie(); + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "URMA QoS integration tests use the link-time mock"; + } + ASSERT_TRUE(urma::IsUrmaAvailable()); + ASSERT_NE(nullptr, urma::GetUrmaContext()); + } + static bool _mock; + static urma::UrmaEndpoint* Endpoint(SocketUniquePtr& socket) { + return static_cast(socket->_transport.get()) + ->GetUrmaEp(); + } + static int Sl2Priority() { + return urma::GetUrmaTpType() == URMA_RTP ? 0 : 6; + } +}; +bool UrmaQosTest::_mock = false; + +TEST_F(UrmaQosTest, service_level_requires_capability_on_the_selected_path) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.service_level = 3; + Channel channel; + if (urma::GetUrmaTpType() == URMA_CTP) { + EXPECT_EQ(-1, channel.Init("127.0.0.1:8123", &options)); + } else { + ASSERT_EQ(0, channel.Init("127.0.0.1:8123", &options)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(channel._server_id, &socket)); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(3, urma::UrmaQosTestPeer::Priority(Endpoint(socket))); + EXPECT_EQ(3, urma::GetUrmaServiceLevelForPriority(3)); + } +} + +TEST_F(UrmaQosTest, + advertises_selected_path_and_rejects_mismatch_before_import) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + SocketId id; + ASSERT_EQ(0, Socket::Create(options, &id)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(id, &socket)); + auto* endpoint = Endpoint(socket); + auto hello = urma::UrmaQosTestPeer::Hello(endpoint); + EXPECT_EQ(static_cast(urma::GetUrmaTpType()), hello.tp_type); + for (auto path : + {URMA_CTP, URMA_RTP, URMA_UTP, static_cast(255)}) { + if (path == urma::GetUrmaTpType()) { + continue; + } + hello.tp_type = static_cast(path); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Import(endpoint, hello)); + EXPECT_EQ(EPROTO, errno); + } + socket->SetFailed(); +} + +TEST_F(UrmaQosTest, rejects_invalid_and_unsupported_priority) { + butil::EndPoint address; + ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &address)); + for (int priority : {-2, 16, 3, 4, 9}) { + if (priority == 3 && urma::GetUrmaTpType() == URMA_RTP) { + continue; + } + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = priority; + Channel channel; + EXPECT_EQ(-1, channel.Init(address, &co)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos = co.urma_qos; + Server server; + EXPECT_EQ(-1, server.Start("127.0.0.1:0", &so)); + if (!co.urma_qos.IsValid()) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + EXPECT_EQ(-1, Socket::Create(options, &sid)); + EXPECT_EQ(EINVAL, errno); + } else { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(EINVAL, errno); + socket->SetFailed(); + } + } +} + +TEST_F(UrmaQosTest, rejects_invalid_unsupported_and_conflicting_service_level) { + struct RequestedQos { + int priority; + int sl; + }; + for (const auto& requested : {RequestedQos{-1, -2}, + {-1, 256}, + {-1, 0}, + {-1, 3}, + {-1, 6}, + {-1, 255}, + {6, 4}, + {3, 3}, + {4, 3}}) { + if (requested.sl == 3 && requested.priority != 4 && + urma::GetUrmaTpType() == URMA_RTP) { + continue; + } + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = requested.priority; + co.urma_qos.service_level = requested.sl; + Channel channel; + EXPECT_EQ(-1, channel.Init("127.0.0.1:8123", &co)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos = co.urma_qos; + Server server; + EXPECT_EQ(-1, server.Start("127.0.0.1:0", &so)); + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + if (!co.urma_qos.IsValid()) { + EXPECT_EQ(-1, Socket::Create(options, &sid)); + EXPECT_EQ(EINVAL, errno); + } else { + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(EINVAL, errno); + socket->SetFailed(); + } + } +} + +TEST_F(UrmaQosTest, + channels_isolate_priority_and_inherit_to_child_connections) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + Channel automatic, low, same, high, named, tcp, tcp_with_qos; + ASSERT_EQ(0, automatic.Init("127.0.0.1:8123", &co)); + co.urma_qos.priority = 6; + ASSERT_EQ(0, low.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, same.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, named.Init("list://127.0.0.1:8123", "rr", &co)); + co.urma_qos.priority = 10; + ASSERT_EQ(0, high.Init("127.0.0.1:8123", &co)); + EXPECT_EQ(low._server_id, same._server_id); + EXPECT_NE(low._server_id, high._server_id); + EXPECT_NE(low._server_id, automatic._server_id); + co.socket_mode = SOCKET_MODE_TCP; + ASSERT_EQ(0, tcp_with_qos.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, tcp.Init("127.0.0.1:8123", nullptr)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); + EXPECT_NE(tcp._server_id, automatic._server_id); + SocketUniquePtr main, pooled, short_socket, selected; + ASSERT_EQ(0, Socket::Address(low._server_id, &main)); + ASSERT_EQ(0, main->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (auto* socket : + {main.get(), pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(6, socket->urma_qos().priority); + } + EXPECT_EQ(main->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); +} + +TEST_F(UrmaQosTest, sub_socket_qos_preserves_common_connection_options) { + brpc::GlobalInitializeOrDie(); + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.priority = 10; + options.urma_qos.service_level = 4; + options.keepalive_options.emplace(); + options.keepalive_options->keepalive_idle_s = 11; + options.keepalive_options->keepalive_interval_s = 12; + options.keepalive_options->keepalive_count = 13; + options.tcp_user_timeout_ms = 1400; + options.bthread_tag = bthread_self_tag() + 1; + SocketId id; + ASSERT_EQ(0, Socket::Create(options, &id)); + BRPC_SCOPE_EXIT { Socket::SetFailed(id); }; + SocketUniquePtr parent; + ASSERT_EQ(0, Socket::Address(id, &parent)); + options.keepalive_options->keepalive_idle_s = 99; + for (int pooled = 0; pooled < 2; ++pooled) { + SCOPED_TRACE(pooled); + SocketUniquePtr child; + ASSERT_EQ(0, pooled ? parent->GetPooledSocket(&child) + : parent->GetShortSocket(&child)); + BRPC_SCOPE_EXIT { child->SetFailed(); }; + EXPECT_EQ(SOCKET_MODE_URMA, child->socket_mode()); + EXPECT_EQ(10, child->urma_qos().priority); + EXPECT_EQ(4, child->urma_qos().service_level); + ASSERT_TRUE(child->_keepalive_options.has_value()); + EXPECT_EQ(11, child->_keepalive_options->keepalive_idle_s); + EXPECT_EQ(12, child->_keepalive_options->keepalive_interval_s); + EXPECT_EQ(13, child->_keepalive_options->keepalive_count); + EXPECT_EQ(1400, child->_tcp_user_timeout_ms); + EXPECT_EQ(options.bthread_tag, child->_io_event.bthread_tag()); + } +} + +TEST_F(UrmaQosTest, service_level_isolates_and_inherits_connections) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.service_level = 2; + Channel low, same, high, named, by_priority, tcp, tcp_with_qos; + ASSERT_EQ(0, low.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, same.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, named.Init("list://127.0.0.1:8123", "rr", &co)); + co.urma_qos.service_level = 4; + ASSERT_EQ(0, high.Init("127.0.0.1:8123", &co)); + co.urma_qos.service_level = -1; + co.urma_qos.priority = 6; + ASSERT_EQ(0, by_priority.Init("127.0.0.1:8123", &co)); + EXPECT_EQ(low._server_id, same._server_id); + EXPECT_NE(low._server_id, high._server_id); + EXPECT_NE(low._server_id, by_priority._server_id); + SocketUniquePtr main, pooled, short_socket, selected; + ASSERT_EQ(0, Socket::Address(low._server_id, &main)); + ASSERT_EQ(0, main->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (auto* socket : + {main.get(), pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(-1, socket->urma_qos().priority); + EXPECT_EQ(2, socket->urma_qos().service_level); + } + EXPECT_EQ(main->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); + co.socket_mode = SOCKET_MODE_TCP; + co.urma_qos.service_level = 256; + ASSERT_EQ(0, tcp_with_qos.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, tcp.Init("127.0.0.1:8123", nullptr)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); +} + +TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { + urma::UrmaEndpoint::GlobalRelease(); + ASSERT_EQ(0, urma::UrmaEndpoint::GlobalInitialize()); + // SL 4 resolves to priority 10 and must leave the default Jettys available. + struct RequestedQos { + int priority; + int sl; + int effective; + }; + for (const auto& qos : {RequestedQos{-1, 4, 10}, + {-1, 2, Sl2Priority()}, + {6, 2, 6}, + {10, -1, 10}, + {-1, -1, urma::GetUrmaJettyPriority()}, + {10, 4, 10}}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.priority = qos.priority; + options.urma_qos.service_level = qos.sl; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = Endpoint(socket); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); + EXPECT_EQ(qos.effective, urma::UrmaQosTestPeer::Priority(ep)); + std::ostringstream debug; + ep->DebugInfo(debug); + EXPECT_NE(std::string::npos, + debug.str().find("urma_effective_priority=")); + EXPECT_NE(std::string::npos, + debug.str().find("urma_sl=" + std::to_string(qos.sl))); + EXPECT_NE(std::string::npos, + debug.str().find( + "urma_effective_sl=" + + std::to_string(urma::GetUrmaServiceLevelForPriority( + qos.effective)))); + socket->SetFailed(); + } +} + +TEST_F(UrmaQosTest, access_error_preserves_priority_and_reports_direction) { + for (int priority : {-1, 10}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.priority = priority; + options.urma_qos.service_level = priority == -1 ? 2 : 4; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = Endpoint(socket); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); + const int effective = priority == -1 ? Sl2Priority() : priority; + for (bool receive : {false, true}) { + urma_cr_t cr{}; + cr.status = URMA_CR_LOC_ACCESS_ERR; + cr.flag.bs.s_r = receive; + cr.local_id = 42; +#if !BRPC_WITH_GLOG + logging::StringSink log; + logging::LogSink* old_sink = logging::SetLogSink(&log); +#endif + const ssize_t result = urma::UrmaQosTestPeer::Complete(ep, cr); + const int saved_errno = errno; +#if !BRPC_WITH_GLOG + logging::SetLogSink(old_sink); + EXPECT_NE(std::string::npos, log.find("status=4")) << log; + EXPECT_NE(std::string::npos, + log.find(receive ? "direction=recv" : "direction=send")) + << log; + EXPECT_NE(std::string::npos, log.find("local_id=42")) << log; + EXPECT_NE(std::string::npos, + log.find("urma_priority=" + std::to_string(priority))) + << log; + EXPECT_NE(std::string::npos, log.find("urma_effective_priority=" + + std::to_string(effective))) + << log; + EXPECT_NE(std::string::npos, + log.find("urma_sl=" + + std::to_string(options.urma_qos.service_level))) + << log; + EXPECT_NE(std::string::npos, + log.find("urma_effective_sl=" + + std::to_string(options.urma_qos.service_level))) + << log; +#endif + EXPECT_EQ(-1, result); + EXPECT_EQ(EIO, saved_errno); + EXPECT_EQ(effective, urma::UrmaQosTestPeer::Priority(ep)); + } + socket->SetFailed(); + } +} + +class UrmaQosEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, + const ::test::EchoRequest* request, + ::test::EchoResponse* response, + google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaQosTest, request_and_response_priorities_apply_across_restart) { + GFLAGS_NAMESPACE::FlagSaver saved_flags; + UrmaQosEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + for (int response_priority : {6, 10}) { + urma::FLAGS_urma_client_handshake_version = + response_priority == 6 ? 2 : 3; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos.priority = response_priority; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = response_priority == 6 ? 10 : 6; + co.timeout_ms = 2000; + co.max_retry = 0; + co.connection_type = "single"; + Channel channel; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("URMA QoS"); + ::test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + SocketUniquePtr client; + ASSERT_EQ(0, Socket::Address(channel._server_id, &client)); + EXPECT_EQ(co.urma_qos.priority, + urma::UrmaQosTestPeer::Priority(Endpoint(client))); + std::vector sockets; + server._am->ListConnections(&sockets); + ASSERT_EQ(1u, sockets.size()); + SocketUniquePtr accepted; + ASSERT_EQ(0, Socket::Address(sockets[0], &accepted)); + EXPECT_EQ(response_priority, accepted->urma_qos().priority); + EXPECT_EQ(response_priority, + urma::UrmaQosTestPeer::Priority(Endpoint(accepted))); + accepted.reset(); + client.reset(); + server.Stop(0); + server.Join(); + } +} + +TEST_F(UrmaQosTest, request_and_response_service_levels_apply_across_restart) { + GFLAGS_NAMESPACE::FlagSaver saved_flags; + UrmaQosEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + for (int response_sl : {2, 4}) { + urma::FLAGS_urma_client_handshake_version = response_sl == 2 ? 2 : 3; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos.service_level = response_sl; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.service_level = response_sl == 2 ? 4 : 2; + co.timeout_ms = 2000; + co.max_retry = 0; + co.connection_type = "single"; + Channel channel; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("URMA SL mapping"); + ::test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + SocketUniquePtr client; + ASSERT_EQ(0, Socket::Address(channel._server_id, &client)); + EXPECT_EQ(-1, client->urma_qos().priority); + EXPECT_EQ(co.urma_qos.service_level, client->urma_qos().service_level); + EXPECT_EQ(response_sl == 2 ? 10 : Sl2Priority(), + urma::UrmaQosTestPeer::Priority(Endpoint(client))); + std::vector sockets; + server._am->ListConnections(&sockets); + ASSERT_EQ(1u, sockets.size()); + SocketUniquePtr accepted; + ASSERT_EQ(0, Socket::Address(sockets[0], &accepted)); + EXPECT_EQ(-1, accepted->urma_qos().priority); + EXPECT_EQ(response_sl, accepted->urma_qos().service_level); + EXPECT_EQ(response_sl == 2 ? Sl2Priority() : 10, + urma::UrmaQosTestPeer::Priority(Endpoint(accepted))); + std::ostringstream debug; + Endpoint(accepted)->DebugInfo(debug); + EXPECT_NE(std::string::npos, + debug.str().find("urma_effective_sl=" + + std::to_string(response_sl))); + accepted.reset(); + client.reset(); + server.Stop(0); + server.Join(); + } +} + +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_recovery_unittest.cpp b/test/brpc_urma_recovery_unittest.cpp new file mode 100644 index 0000000000..afb996b154 --- /dev/null +++ b/test/brpc_urma_recovery_unittest.cpp @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#if BRPC_WITH_URMA + +#include +#include +#include +#include +#include "brpc/urma/urma_recovery_policy.h" + +namespace brpc { +namespace urma { +namespace { + +std::atomic now_us(1000000); +int64_t FakeClock() { return now_us.load(); } + +class UrmaRecoveryTest : public testing::Test { +protected: + UrmaRecoveryTest() : policy(FakeClock, 2) {} + void SetUp() override { now_us = 1000000; } + UrmaRecoveryPolicy policy; +}; + +TEST_F(UrmaRecoveryTest, legacy_preserves_admission) { + uint64_t epoch; + EXPECT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.SetMode(UrmaRecoveryMode::AUTO)); + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(UrmaRecoveryTest, peer_failure_only_cools_that_peer) { + policy.SetMode(UrmaRecoveryMode::AUTO); + policy.SetCooldownMs(10); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("")); + now_us += 9999; + EXPECT_FALSE(policy.AllowNewConnection("a")); + ++now_us; + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(UrmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { + policy.SetMode(UrmaRecoveryMode::AUTO); + uint64_t old_epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &old_epoch)); + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); + policy.SetMode(UrmaRecoveryMode::AUTO); + policy.ReportPeerFailure("a", old_epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + EXPECT_NE(old_epoch, epoch); + // Reapplying AUTO does not clear existing quarantine. + policy.ReportPeerFailure("a", epoch); + policy.SetMode(UrmaRecoveryMode::AUTO); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(UrmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { + policy.SetMode(UrmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + ++now_us; + policy.ReportPeerFailure("b", epoch); + ++now_us; + policy.ReportPeerFailure("c", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_FALSE(policy.AllowNewConnection("c")); + now_us += 5000000; + policy.ReportPeerFailure("d", epoch); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("c")); + EXPECT_FALSE(policy.AllowNewConnection("d")); +} + +TEST_F(UrmaRecoveryTest, validates_configuration) { + EXPECT_FALSE(policy.SetMode(static_cast(-1))); + EXPECT_EQ(UrmaRecoveryMode::LEGACY, policy.GetMode()); + EXPECT_FALSE(policy.SetCooldownMs(0)); + EXPECT_FALSE(policy.SetCooldownMs(-1)); + EXPECT_FALSE(policy.SetCooldownMs(static_cast(INT32_MAX) + 1)); + EXPECT_TRUE(policy.SetCooldownMs(INT32_MAX)); + policy.SetMode(UrmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(UrmaRecoveryTest, concurrent_admission_failure_and_mode_change) { + std::vector workers; + for (int i = 0; i < 8; ++i) { + workers.emplace_back([this, i] { + for (int j = 0; j < 200; ++j) { + uint64_t epoch; + const std::string peer = std::to_string(i); + policy.SetMode(j % 2 ? UrmaRecoveryMode::AUTO + : UrmaRecoveryMode::TCP_ONLY); + policy.AllowNewConnection(peer, &epoch); + policy.ReportPeerFailure(peer, epoch); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); +} + +TEST_F(UrmaRecoveryTest, reports_reasons_and_admission_counters) { + policy.SetMode(UrmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.RecordNegotiatedConnection(epoch); + policy.ReportPeerFailure("a", epoch, UrmaFailureReason::QP_FATAL); + EXPECT_FALSE(policy.AllowNewConnection("a")); + auto status = policy.GetStatus(); + EXPECT_EQ(1u, status.cooled_peers); + EXPECT_EQ(1u, status.negotiated_connections); + EXPECT_EQ(1u, status.peer_cooldown_admissions); + EXPECT_EQ( + 1u, status.failures[static_cast(UrmaFailureReason::QP_FATAL)]); + policy.ReportPeerFailure("b", epoch, UrmaFailureReason::ACCESS_ERROR); + EXPECT_TRUE(policy.AllowNewConnection("b")); + now_us += 5000000; + EXPECT_EQ(0u, policy.GetStatus().cooled_peers); + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_EQ(1u, policy.GetStatus().tcp_only_admissions); + policy.ReportPeerFailure("b", epoch, UrmaFailureReason::CQ_ERROR); + EXPECT_EQ(0u, + policy.GetStatus() + .failures[static_cast(UrmaFailureReason::CQ_ERROR)]); +} + +} // namespace +} // namespace urma +} // namespace brpc + +#endif // BRPC_WITH_URMA diff --git a/test/brpc_urma_remote_recovery_unittest.cpp b/test/brpc_urma_remote_recovery_unittest.cpp new file mode 100644 index 0000000000..1e97f5d1fe --- /dev/null +++ b/test/brpc_urma_remote_recovery_unittest.cpp @@ -0,0 +1,208 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#if BRPC_WITH_URMA +#include +#include +#include +#include +#include +#include + +#include "brpc/channel.h" +#include "brpc/closure_guard.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "bthread/bthread.h" +#include "butil/time.h" +#include "butil/logging.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class RemoteFailureEcho : public test::EchoService { +public: + std::string marker; + std::atomic fail_next_response{false}; + + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + // The request has arrived; the next data SEND belongs to the server. + if (fail_next_response.exchange(false)) { + FILE* file = fopen(marker.c_str(), "w"); + CHECK(file != nullptr); + CHECK_EQ(0, fclose(file)); + } + } +}; + +class UrmaRemoteRecoveryTest : public testing::Test { +protected: + void SetUp() override { + const char* preload = std::getenv("LD_PRELOAD"); + if (!preload || + !std::strstr(preload, "brpc_urma_send_failure_injector")) { + GTEST_SKIP() << "Run through CTest to preload the SEND injector"; + } + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + int count = 0; + auto** devices = urma_get_device_list(&count); + const bool mock = + devices && count > 0 && + std::strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + ASSERT_EQ(URMA_SUCCESS, urma_uninit()); + } + if (!mock) { + GTEST_SKIP() << "Remote failure tests use the URMA mock"; + } + brpc::urma::FLAGS_urma_buffer_count = 4096; + brpc::urma::GlobalUrmaInitializeOrDie(); + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + ASSERT_TRUE(brpc::urma::SetUrmaRecoveryCooldownMs(2000)); + ASSERT_TRUE(brpc::urma::SetUrmaRecoveryMode( + brpc::urma::UrmaRecoveryMode::AUTO)); + char marker[] = "/tmp/brpc-urma-remote-XXXXXX"; + const int fd = mkstemp(marker); + ASSERT_GE(fd, 0); + ASSERT_EQ(0, close(fd)); + _service.marker = marker; + ASSERT_EQ(0, unlink(marker)); + ASSERT_EQ(0, setenv("BRPC_URMA_FAIL_ONCE_FILE", marker, 1)); + ASSERT_EQ( + 0, _server.AddService(&_service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions options; + options.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, _server.Start("127.0.0.1:0", &options)); + } + + void TearDown() override { + _server.Stop(0); + _server.Join(); + if (!_service.marker.empty()) { + unlink(_service.marker.c_str()); + unsetenv("BRPC_URMA_FAIL_ONCE_FILE"); + } + brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::LEGACY); + } + + bool Echo(brpc::Channel* channel) { + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message("remote failure recovery"); + const std::string payload(65536, 'r'); + cntl.request_attachment().append(payload); + test::EchoService_Stub(channel).Echo(&cntl, &request, &response, + nullptr); + if (cntl.Failed()) { + return false; + } + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + return true; + } + + template + bool WaitUntil(Predicate predicate) { + const int64_t deadline = butil::monotonic_time_us() + 3000000; + while (!predicate()) { + if (butil::monotonic_time_us() >= deadline) { + return false; + } + bthread_usleep(1000); + } + return true; + } + + RemoteFailureEcho _service; + brpc::Server _server; +}; + +TEST_F(UrmaRemoteRecoveryTest, server_send_failure_cools_client_and_recovers) { + brpc::Channel channel; + brpc::ChannelOptions options; + options.socket_mode = brpc::SOCKET_MODE_URMA; + options.connection_type = "short"; + options.timeout_ms = 1000; + options.connect_timeout_ms = 1000; + options.max_retry = 0; + ASSERT_EQ(0, channel.Init(_server.listen_address(), &options)); + const auto transport = + static_cast(brpc::urma::UrmaFailureReason::TRANSPORT_ERROR); + const auto send_post = + static_cast(brpc::urma::UrmaFailureReason::SEND_POST); + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + // Normal client-initiated short closes must not quarantine the server. + for (int i = 0; i < 8; ++i) { + ASSERT_TRUE(Echo(&channel)); + } + EXPECT_EQ(0u, brpc::urma::GetUrmaRecoveryStatus().cooled_peers); + EXPECT_EQ(before.failures[transport], + brpc::urma::GetUrmaRecoveryStatus().failures[transport]); + + _service.fail_next_response = true; + EXPECT_FALSE(Echo(&channel)); + ASSERT_EQ(before.failures[send_post] + 1, + brpc::urma::GetUrmaRecoveryStatus().failures[send_post]); + ASSERT_TRUE(WaitUntil( + [] { return brpc::urma::GetUrmaRecoveryStatus().cooled_peers == 1; })); + const auto cooling = brpc::urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(before.failures[transport] + 1, cooling.failures[transport]); + ASSERT_TRUE(Echo(&channel)); + const auto tcp = brpc::urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(cooling.negotiated_connections, tcp.negotiated_connections); + EXPECT_GT(tcp.peer_cooldown_admissions, cooling.peer_cooldown_admissions); + + ASSERT_TRUE(WaitUntil( + [] { return brpc::urma::GetUrmaRecoveryStatus().cooled_peers == 0; })); + ASSERT_TRUE(Echo(&channel)); + const auto recovered = brpc::urma::GetUrmaRecoveryStatus(); + EXPECT_GT(recovered.negotiated_connections, tcp.negotiated_connections); + EXPECT_EQ(tcp.peer_cooldown_admissions, recovered.peer_cooldown_admissions); + EXPECT_EQ(before.failures[send_post] + 1, recovered.failures[send_post]); + EXPECT_EQ(before.failures[transport] + 1, recovered.failures[transport]); +} + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_runtime_unittest.cpp b/test/brpc_urma_runtime_unittest.cpp new file mode 100644 index 0000000000..cd6cc998b9 --- /dev/null +++ b/test/brpc_urma_runtime_unittest.cpp @@ -0,0 +1,671 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "butil/atomicops.h" +#include "butil/sys_byteorder.h" +#include "urma_api.h" +#include "brpc/urma/urma_handshake.h" +#include "brpc/urma/urma_handshake.pb.h" +#include "brpc/urma/urma_helper.h" +#include "urma_types.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/fd_guard.h" +#include "bvar/variable.h" +#include "echo.pb.h" + +using namespace brpc; + +namespace brpc { +namespace urma { + +DECLARE_int32(urma_client_handshake_version); +extern bool g_skip_urma_init; +extern butil::atomic g_urma_available; + +} // namespace urma +} // namespace brpc + +TEST(UrmaRuntimeHelperTest, combined_priority_and_invalid_transport_type) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( + attr, static_cast(255))); +} + +TEST(UrmaRuntimeHelperTest, + auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); + attr.dev_cap.priority_info[7].tp_type.bs.ctp = 1; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_CTP, path); + EXPECT_EQ(7, priority); +} + +TEST(UrmaRuntimeHelperTest, + explicit_path_uses_matching_priority_and_combined_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[2].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[2].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"rtp", "ctp"}) { + ASSERT_TRUE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + EXPECT_EQ(std::string(requested) == "rtp" ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(2, priority); + } +} + +TEST(UrmaRuntimeHelperTest, + rejects_unsupported_paths_in_a_reported_capability_table) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "ctp", &path, &priority)); + EXPECT_EQ(URMA_UTP, path); + EXPECT_EQ(15, priority); + attr.dev_cap.priority_info[0].tp_type.value = 0; + attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); +} + +TEST(UrmaRuntimeHelperTest, + providers_without_path_capabilities_keep_compatibility_priorities) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"auto", "ctp", "rtp"}) { + ASSERT_TRUE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + const bool rtp = std::string(requested) == "rtp"; + EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(rtp ? 0 : 6, priority); + } +} + +TEST(UrmaRuntimeHelperTest, rejects_invalid_path_configuration) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + for (const auto& requested : {"", "utp", "RM", "typo"}) { + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + } + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, nullptr)); +} + +TEST(UrmaRuntimeHelperTest, + local_sge_initializes_optional_fields_and_accepts_segment_edges) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge; + memset(&sge, 0xff, sizeof(sge)); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x1000, 8192, &segment)); + EXPECT_EQ(0x1000u, sge.addr); + EXPECT_EQ(8192u, sge.len); + EXPECT_EQ(&segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x2fff, 1, &segment)); + EXPECT_EQ(0x2fffu, sge.addr); + EXPECT_EQ(1u, sge.len); +} + +TEST(UrmaRuntimeHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge{}; + for (auto values : {std::pair(0xfff, 1), + {0x3000, 1}, + {0x2fff, 2}, + {0x1000, 0}, + {0x1000, static_cast(UINT32_MAX) + 1}}) { + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, + &segment)); + EXPECT_EQ(ERDMAMEM, errno); + } + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); + segment.urma_ctx = reinterpret_cast(1); + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, &segment)); + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = UINT64_MAX - 7; + segment.seg.len = 16; + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); +} + +static urma_target_seg_t g_copy_test_segment; +static int g_copy_test_releases; + +static urma_target_seg_t* ResolveCopyTestSegment(void* address) { + g_copy_test_segment = urma_target_seg_t{}; + g_copy_test_segment.urma_ctx = urma::GetUrmaContext(); + g_copy_test_segment.seg.ubva.va = reinterpret_cast(address); + g_copy_test_segment.seg.len = urma::GetUrmaRecvBlockSize(); + return &g_copy_test_segment; +} + +static urma_target_seg_t* RejectCopyTestSegment(void*) { return nullptr; } + +static void DeleteCopyTestData(void* data) { + ++g_copy_test_releases; + delete[] static_cast(data); +} + +TEST(UrmaRuntimeHelperTest, + copied_send_coalesces_host_fragments_and_retains_aligned_payload) { + g_copy_test_releases = 0; + butil::IOBuf buffer; + for (const char* text : {"first", "second"}) { + const size_t length = strlen(text); + char* data = new char[length]; + memcpy(data, text, length); + ASSERT_EQ(0, buffer.append_user_data(data, length, DeleteCopyTestData)); + } + ASSERT_EQ(2u, buffer.backing_block_num()); + urma_sge_t sge{}; + ASSERT_TRUE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(2, g_copy_test_releases); + EXPECT_EQ("firstsecond", buffer.to_string()); + ASSERT_EQ(1u, buffer.backing_block_num()); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), + sge.addr); + EXPECT_EQ(buffer.size(), sge.len); + EXPECT_EQ(&g_copy_test_segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + EXPECT_EQ("firstsecond", + std::string(reinterpret_cast(sge.addr), sge.len)); + + buffer.clear(); + const std::string large( + urma::GetUrmaRecvBlockSize() - (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), + 'x'); + buffer.append(large); + ASSERT_TRUE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(large, buffer.to_string()); + EXPECT_EQ(large.size(), sge.len); +} + +TEST(UrmaRuntimeHelperTest, copied_send_failure_preserves_source_payload) { + butil::IOBuf buffer; + buffer.append("unchanged"); + urma_sge_t sge{}; + EXPECT_FALSE( + urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); + EXPECT_EQ(ERDMAMEM, errno); + EXPECT_EQ("unchanged", buffer.to_string()); + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, nullptr)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ("unchanged", buffer.to_string()); + const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); + buffer.clear(); + buffer.append(large); + EXPECT_FALSE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(EMSGSIZE, errno); + EXPECT_EQ(large, buffer.to_string()); +} + +class UrmaRuntimeMockTest : public ::testing::Test { +protected: + void SetUp() override { + urma::g_skip_urma_init = false; + + urma_init_attr_t init_attr{}; + const urma_status_t status = urma_init(&init_attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + _owns_urma_init = status == URMA_SUCCESS; + + int num_devices = 0; + urma_device_t** devices = urma_get_device_list(&num_devices); + ASSERT_NE(nullptr, devices); + ASSERT_GT(num_devices, 0); + const bool using_mock = + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (!using_mock) { + if (_owns_urma_init) { + EXPECT_EQ(URMA_SUCCESS, urma_uninit()); + _owns_urma_init = false; + } + GTEST_SKIP() + << "UrmaRuntimeMockTest requires the URMA link-time mock"; + } + } + + void TearDown() override { + if (_owns_urma_init) { + EXPECT_EQ(URMA_SUCCESS, urma_uninit()); + } + urma::g_skip_urma_init = true; + urma::g_urma_available.store(true, butil::memory_order_relaxed); + } + +private: + bool _owns_urma_init{false}; +}; + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static int ImportPeer(UrmaEndpoint* ep, const ParsedHello& peer) { + return ep->ImportPeer(peer); + } + static void Establish(UrmaEndpoint* ep) { + uint64_t epoch = 0; + GlobalUrmaRecoveryPolicy().AllowNewConnection("", &epoch); + ep->_recovery_epoch.store(epoch, butil::memory_order_relaxed); + ep->_state = UrmaEndpoint::ESTABLISHED; + } + static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { + urma_cr_t cr{}; + cr.status = status; + return ep->HandleCompletion(cr); + } + static void Report(UrmaEndpoint* ep, UrmaFailureReason reason) { + ep->ReportFailure(reason); + } + static void PollFailure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jfc_t invalid_jfc{}; + resource.jfc = &invalid_jfc; + ep->_resource = &resource; + SocketOptions options; + SocketId sid; + if (Socket::Create(options, &sid) == 0) { + SocketUniquePtr carrier; + if (Socket::Address(sid, &carrier) == 0) { + const bool polling = FLAGS_urma_use_polling; + FLAGS_urma_use_polling = true; + SocketUniquePtr owner; + if (Socket::Address(ep->_socket->id(), &owner) == 0) { + ep->_cq_sid = sid; + ep->DrainCq(carrier.get(), owner); + ep->_cq_sid = INVALID_SOCKET_ID; + } + FLAGS_urma_use_polling = polling; + carrier->SetFailed(); + } + } + ep->_resource = nullptr; + resource.jfc = nullptr; + } + static bool Backpressure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_window_size = 0; + ep->_remote_rq_window_size = 1; + butil::IOBuf data; + data.append("payload"); + butil::IOBuf* buf = &data; + const ssize_t rc = ep->CutFromIOBufList(&buf, 1); + const bool ok = rc == -1 && errno == EAGAIN && data.size() == 7; + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return ok; + } + static int FailedAck(UrmaEndpoint* ep) { + // A Jetty without an imported target fails provider submission. + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_imm_window_size = 1; + ep->_remote_window_capacity = 2; + ep->_new_rq_wrs = 2; + urma_cr_t cr{}; + cr.status = URMA_CR_SUCCESS; + cr.user_ctx = 0; + const int rc = ep->HandleCompletion(cr); + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return rc; + } +}; + +} // namespace urma +} // namespace brpc + +class UrmaRecoveryTest : public UrmaRuntimeMockTest { +protected: + void SetUp() override { + UrmaRuntimeMockTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + urma::g_skip_urma_init = true; + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + } + void TearDown() override { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + UrmaRuntimeMockTest::TearDown(); + } + void CreateSocket(SocketUniquePtr* socket) { + SocketOptions opt; + butil::str2endpoint("127.0.0.1:8123", &opt.remote_side); + opt.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(opt, &sid)); + ASSERT_EQ(0, Socket::Address(sid, socket)); + } +}; + +TEST_F(UrmaRecoveryTest, recovery_mode_and_metrics) { + ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); + EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); + EXPECT_NE(std::string::npos, + bvar::Variable::describe_exposed("urma_recovery_status") + .find("tcp_only")); + EXPECT_FALSE(bvar::Variable::describe_exposed( + "urma_recovery_tcp_fallback_admissions") + .empty()); +} + +TEST_F(UrmaRecoveryTest, rejects_mismatched_path_before_importing_resources) { + urma::UrmaEndpoint endpoint(nullptr); + urma::ParsedHello peer; + peer.tp_type = urma::GetUrmaTpType() == URMA_CTP ? URMA_RTP : URMA_CTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); + peer.tp_type = URMA_UTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); +} + +TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { + for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, + URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); + const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR + ? urma::UrmaFailureReason::RETRY_EXHAUSTED + : status == URMA_CR_RNR_RETRY_CNT_EXC_ERR + ? urma::UrmaFailureReason::RNR + : status == URMA_CR_LOC_ACCESS_ERR + ? urma::UrmaFailureReason::ACCESS_ERROR + : urma::UrmaFailureReason::TRANSPORT_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + urma::GetUrmaRecoveryStatus() + .failures[static_cast(reason)]); + EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection( + "127.0.0.1:8123")); + urma::UrmaRecoveryTestPeer::Report(&ep, + urma::UrmaFailureReason::CQ_POLL); + EXPECT_EQ(before.failures[static_cast( + urma::UrmaFailureReason::CQ_POLL)], + urma::GetUrmaRecoveryStatus().failures[static_cast( + urma::UrmaFailureReason::CQ_POLL)]); + socket->SetFailed(); + } +} + +TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); + EXPECT_EQ(-1, + urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); + EXPECT_TRUE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_TRUE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); + EXPECT_FALSE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + urma::UrmaRecoveryTestPeer::PollFailure(&ep); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ( + before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + + 1, + urma::GetUrmaRecoveryStatus() + .failures[static_cast(urma::UrmaFailureReason::CQ_POLL)]); +} + +class UrmaRecoveryEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, + const ::test::EchoRequest* request, + ::test::EchoResponse* response, + google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + UrmaRecoveryEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + Channel channel; + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 2000; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + ::test::EchoService_Stub stub(&channel); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("recovery over TCP"); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketOptions failed_options; + failed_options.remote_side = server.listen_address(); + failed_options.user = get_client_side_messenger(); + SocketId failed_sid; + ASSERT_EQ(0, Socket::Create(failed_options, &failed_sid)); + SocketUniquePtr failed_socket; + ASSERT_EQ(0, Socket::Address(failed_sid, &failed_socket)); + { + urma::UrmaEndpoint failed_ep(failed_socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&failed_ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( + &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); + } + failed_socket->SetFailed(); + const auto before = urma::GetUrmaRecoveryStatus(); + cntl.Reset(); + response.Clear(); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(before.peer_cooldown_admissions + 1, + urma::GetUrmaRecoveryStatus().peer_cooldown_admissions); + server.Stop(0); + server.Join(); +} + +TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + Server server; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + for (int version : {2, 3}) { + butil::fd_guard fd( + butil::tcp_connect(server.listen_address(), nullptr)); + ASSERT_GE(fd, 0); + timeval timeout{2, 0}; + ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout))); + urma::v2_wire::HelloMessage hello; + hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; + hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; + hello.impl_ver = urma::v2_wire::IMPL_V2_VERSION; + hello.buffer_size = 8192; + hello.recv_buffer_cnt = 127; + hello.jetty_id = 42; + memset(hello.eid, 1, 16); + memset(hello.seg_eid, 1, 16); + hello.tp_type = URMA_CTP; + hello.seg_va = 0x100000; + hello.seg_len = 1024 * 1024; + if (version == 2) { + uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; + memcpy(packet, "URMA", 4); + hello.Serialize(packet + 4); + ASSERT_EQ(sizeof(packet), + static_cast(write(fd, packet, sizeof(packet)))); + uint8_t reply[sizeof(packet)]; + ASSERT_EQ(sizeof(reply), + static_cast( + recv(fd, reply, sizeof(reply), MSG_WAITALL))); + EXPECT_EQ(0, memcmp(reply, "URMA", 4)); + urma::v2_wire::HelloMessage rejected; + rejected.Deserialize(reply + 4); + EXPECT_EQ(0, rejected.hello_ver); + EXPECT_EQ(0u, rejected.jetty_id); + } else { + urma::UrmaHello msg; + msg.set_buffer_size(8192); + msg.set_recv_buffer_cnt(127); + msg.set_jetty_id(42); + msg.set_uasid(0); + msg.set_seg_uasid(0); + msg.set_seg_token_id(0); + msg.set_eid(hello.eid, 16); + msg.set_seg_eid(hello.seg_eid, 16); + msg.set_tp_type(URMA_CTP); + msg.set_seg_va(hello.seg_va); + msg.set_seg_len(hello.seg_len); + const std::string body = msg.SerializeAsString(); + uint32_t size = butil::HostToNet32(body.size()); + std::string packet("URM3", 4); + packet.append(reinterpret_cast(&size), 4); + packet.append(body); + ASSERT_EQ(packet.size(), static_cast(write( + fd, packet.data(), packet.size()))); + char header[8]; + ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); + EXPECT_EQ(0, memcmp(header, "URM3", 4)); + memcpy(&size, header + 4, 4); + size = butil::NetToHost32(size); + ASSERT_LE(size, 4096u); + std::string reply(size, '\0'); + ASSERT_EQ(size, static_cast( + recv(fd, &reply[0], size, MSG_WAITALL))); + urma::UrmaHello rejected; + ASSERT_TRUE(rejected.ParseFromString(reply)); + EXPECT_EQ(0u, rejected.jetty_id()); + } + const std::string request = + "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), + request.size()))); + char response[4096]; + const ssize_t nr = recv(fd, response, sizeof(response), 0); + ASSERT_GT(nr, 0); + EXPECT_NE(std::string::npos, std::string(response, nr).find(" 200 ")); + } + server.Stop(0); + server.Join(); +} + +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); +#if BRPC_WITH_URMA + urma::g_skip_urma_init = true; + urma::g_urma_available.store(true, butil::memory_order_relaxed); +#endif + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_transport_unittest.cpp b/test/brpc_urma_transport_unittest.cpp new file mode 100644 index 0000000000..e1193fde0f --- /dev/null +++ b/test/brpc_urma_transport_unittest.cpp @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_handshake.h" +#include "brpc/urma/urma_handshake.pb.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_string(urma_tp_type); +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class PathEchoService : public test::EchoService { + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +class UrmaTransportTest : public ::testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (_mock) { + brpc::urma::FLAGS_urma_buffer_count = 4096; + brpc::urma::GlobalUrmaInitializeOrDie(); + } + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "Path RPC tests use the link-time URMA mock"; + } + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + const bool rtp = brpc::urma::FLAGS_urma_tp_type == "rtp"; + ASSERT_EQ(rtp ? URMA_RTP : URMA_CTP, brpc::urma::GetUrmaTpType()); + ASSERT_EQ(rtp ? 0 : 6, brpc::urma::GetUrmaJettyPriority()); + brpc::urma::FLAGS_urma_client_handshake_version = GetParam(); + brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); + } + void TearDown() override { + brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::LEGACY); + } + static bool _mock; +}; +bool UrmaTransportTest::_mock = false; + +TEST_P(UrmaTransportTest, advertises_selected_path_in_both_hello_formats) { + brpc::urma::UrmaEndpoint endpoint(nullptr); + brpc::urma::v2_wire::HelloMessage hello; + endpoint.FillLocalHelloV2(&hello); + EXPECT_EQ(brpc::urma::GetUrmaTpType(), hello.tp_type); + brpc::urma::UrmaHello protobuf_hello; + endpoint.FillLocalHelloV3(&protobuf_hello); + EXPECT_EQ(brpc::urma::GetUrmaTpType(), protobuf_hello.tp_type()); +} + +TEST_P(UrmaTransportTest, exchanges_zero_copy_payloads_and_receive_credits) { + PathEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "single"; + co.timeout_ms = 3000; + co.max_retry = 0; + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + for (int round = 0; round < 40; ++round) { + const size_t size = round == 0 ? 0 : round == 1 ? 1024 * 1024 : 20000; + const std::string payload(size, 'r'); + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message(payload); + cntl.request_attachment().append(payload); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(payload, response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + } + EXPECT_EQ(before.negotiated_connections + 2, + brpc::urma::GetUrmaRecoveryStatus().negotiated_connections); + server.Stop(0); + server.Join(); +} + +TEST_P(UrmaTransportTest, + switches_new_connections_from_auto_to_tcp_only_and_back) { + PathEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 3000; + co.max_retry = 0; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + // Cumulative receive buffers exceed the pool capacity unless closed + // short connections release their queues and buffers. + const int requests_per_mode = 40; + for (auto mode : {brpc::urma::UrmaRecoveryMode::AUTO, + brpc::urma::UrmaRecoveryMode::TCP_ONLY, + brpc::urma::UrmaRecoveryMode::AUTO}) { + ASSERT_TRUE(brpc::urma::SetUrmaRecoveryMode(mode)); + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + for (int i = 0; i < requests_per_mode; ++i) { + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message("switch"); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ("switch", response.message()); + } + const auto after = brpc::urma::GetUrmaRecoveryStatus(); + if (mode == brpc::urma::UrmaRecoveryMode::TCP_ONLY) { + EXPECT_EQ(before.negotiated_connections, + after.negotiated_connections); + EXPECT_EQ(before.tcp_only_admissions + requests_per_mode, + after.tcp_only_admissions); + } else { + EXPECT_EQ(before.negotiated_connections + 2 * requests_per_mode, + after.negotiated_connections); + EXPECT_EQ(before.tcp_only_admissions, after.tcp_only_admissions); + } + } + server.Stop(0); + server.Join(); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaTransportTest, + ::testing::Values(2, 3)); + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +}