From 45e08065d0d9b33cfef2f9834d3bfb6c0fff53f1 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:22:22 -0700 Subject: [PATCH 01/29] Design RDMA runtime recovery and controlled TCP fallback --- docs/cn/rdma.md | 5 + docs/cn/rdma_runtime_recovery.md | 231 +++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 docs/cn/rdma_runtime_recovery.md diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index fd70686bb1..96fa00a103 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,3 +1,8 @@ +# 运行期故障恢复设计 + +[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md) 为尚未实现的设计草案, +讨论端口故障隔离、冷却与恢复探测、运行时降级控制及多端口主备。 + # 编译 由于RDMA对驱动与硬件有要求,目前仅支持在Linux系统编译并运行RDMA功能。 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md new file mode 100644 index 0000000000..8e1af1ef47 --- /dev/null +++ b/docs/cn/rdma_runtime_recovery.md @@ -0,0 +1,231 @@ + + +# RDMA 运行期故障恢复与 TCP 降级设计 + +状态:设计草案,尚未实现。本文中的接口、参数和指标均为提案。 + +基线为 LinQuickDev/brpc master 的 +`87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 +本提案改变后续连接的传输选择;不承诺存量 RPC 无损迁移或 exactly-once。 + +## 问题与源码依据 + +当前握手可以回落 TCP,但 RDMA 建立后的故障主要通过连接失败处理: + +| 位置 | 当前行为 | 需要补充的能力 | +| --- | --- | --- | +| `RdmaConnect::StartConnect`、握手状态机 | RDMA 不可用或协商失败时回落 TCP | 后续建链能记住运行期故障并绕过故障路径 | +| `RdmaEndpoint::PollCq` | 错误 CQE 调用 `Socket::SetFailed` | 错误分类、隔离和恢复探测 | +| `OnRdmaAsyncEvent` | PORT_ERR / PORT_ACTIVE 只确认事件 | 端口状态与连接准入联动 | +| `GlobalDisableRdma` | 修改可用状态并记录 FATAL | 独立、非致命的运维降级控制 | +| `CutFromIOBufList` | 窗口耗尽返回 EAGAIN;post_send 失败按不可恢复处理 | 保留背压语义,检测有未完成发送时的长期无进展 | +| `BringUpQp` | timeout=14、retry_cnt=7、rnr_retry=0 | 明确错误语义及受限配置,而非把所有错误归为超时 | + +源码分别位于 `src/brpc/rdma/rdma_endpoint.cpp`、 +`src/brpc/rdma/rdma_helper.cpp` 和 `src/brpc/rdma_transport.cpp`。 +RPC deadline、verbs ACK 重试耗尽、RNR 重试耗尽和本地 SQ 背压是不同事件。 + +## 目标和兼容性 + +1. 本地 RDMA 端口故障后,快速停止该端口的新建链并使相关连接失败。 +2. 后续连接优先使用可达的备用 RDMA 路径,无可用路径时通过新 TCP 连接恢复。 +3. 对故障路径设置冷却和有界恢复探测,避免每次 RPC 都重新尝试故障 RDMA。 +4. 提供运行时 TCP-only 开关,并明确存量连接的处理方式。 +5. 给出 SQ 停顿、RNR 和链路错误的独立策略、指标和测试标准。 + +自动恢复默认关闭;启用前保留现有单设备、单端口和握手协议行为。 +TCP fallback 遵循现有协议与 SSL 限制,不绕过认证,不隐式降低安全配置。 +发布第一阶段只实现单设备故障隔离和 TCP 恢复;跨 HCA 主备在资源模型改造后提供。 + +## 恢复边界 + +故障连接执行正常 SetFailed、唤醒写等待者并进入回收流程。 +策略先发布故障状态,再传播连接失败,使同时发生的新建链看到最新路由。 +已经建立的 RDMA 连接不改为在原 TCP fd 上发送 RPC 数据。 +该 fd 在握手后有独立语义,直接切换会造成消息边界、解析状态和顺序问题。 + +只由现有 Controller、RetryPolicy、max_retry 和 RPC deadline 决定是否重试。 +新增策略不增加重试配额、不重置 deadline、不自动重放已 post 的 WR。 +请求可能已被服务端执行时保留现有错误语义;有副作用的业务仍需幂等键或去重。 +恢复策略只保证后续连接避开故障 RDMA,不保证正在执行的调用成功。 + +## 组件与资源所有权 + +在 `src/brpc/rdma/` 新增 `RdmaRecoveryPolicy`,管理准入、冷却和探测。 +`RdmaTransport` 在建链前获取一次不可变的 route lease; +`RdmaEndpoint` 的 QP、CQ、GID、port 和注册内存绑定到该 lease。 + +状态分为两个作用域: + +- 本地端口键:device identity、port number、资源 generation。 +- 对端路径键:本地端口键、远端地址、传输模式、协商相关身份。 + +QP 错误先影响连接或对端路径,不根据一个对端失败就禁用整块网卡。 +新连接可读取端口状态和对端状态的不可变快照;数据发送热路径不加全局锁。 +有界缓存使用过期时间和容量限制;只逐出无活动 lease、无探测的对端状态。 +端口配置固定在启动时,运维开关在运行时更新策略 epoch。 + +## 状态机与故障分类 + +端口和对端路径分别维护 HEALTHY、COOLDOWN、PROBING、DISABLED。 +单调时钟驱动冷却;每个状态更新带 generation 和 policy epoch。 + +| 输入 | 状态变化 | 行为 | +| --- | --- | --- | +| 本地 PORT_ERR | 端口进入 COOLDOWN | 禁止新 RDMA;使绑定该 generation 的连接失败 | +| PORT_ACTIVE | 保留隔离,记录链路恢复 | 不立即恢复全量流量,等待冷却和探测 | +| QP/CQ 错误 | 连接失败,必要时对端路径进入 COOLDOWN | 不据此断定本地端口故障 | +| RETRY_EXC | 对端路径进入 COOLDOWN | 原因可能是路径、对端或网络;不累计为本地端口故障 | +| RNR_RETRY_EXC | 对端路径进入 COOLDOWN | 记录接收资源/流控异常,不切坏整个本地端口 | +| 正常 SQ/RQ 窗口为零 | 不改变健康状态 | EAGAIN 背压,等待 completion / credit | +| post_send 失败 | 连接失败,按 errno 分类 | 保存原始错误;不直接重投 WR 或循环忙重试 | +| 运维 TCP-only | 准入进入 DISABLED | 新建链直接选择 TCP,按下文处理存量连接 | +| DEVICE_FATAL | 对应资源域永久 DISABLED | 仅在安全排空得到验证后支持非致命降级,否则保留现有 FATAL | + +禁止把 CQ flush 连锁错误重复计为多次独立故障。 +同一 socket generation 的失败使用一次性报告,过期事件不得修改新一代状态。 +内存注册错误、远端访问错误和本地保护错误保留独立诊断;不以自动重试掩盖编程错误。 + +## 冷却与恢复探测 + +首次隔离使用基础冷却时间,连续探测失败指数退避至上限,并加入有界 jitter。 +冷却期所有新连接跳过该路径,选择健康候选路径或 TCP。 +冷却结束不等于恢复:原子准入最多一个新 RDMA 连接作为探测,其余请求继续走 TCP。 +PORT_ERR 后还必须先确认本地端口 ACTIVE,才能发放探测 lease。 + +握手成功只能证明建链成功。探测连接收到有效 RDMA 数据、且至少一次发送获得完成后, +才把路径恢复为 HEALTHY;不能以 TCP 成功证明 RDMA 已恢复。 +不构造会产生业务副作用的探测 RPC;使用正常业务流量并受原调用 deadline 约束。 +探测在预算内失败或没有足够证据时重新冷却,不长期占用探测名额。 +一条探测连接同时持有端口和对端 lease,成功/失败只释放对应 epoch 的名额。 + +## 运行时控制与连接复用 + +提议 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 及进程级原子 setter/getter。 +setter 校验输入、增加 epoch,并唤醒控制任务;不直接销毁 QP 或 MR。 +运维系统可调用 API,初版不新增可远程修改状态的 HTTP 接口。 + +TCP_ONLY 的完成语义是“停止新 RDMA 准入并返回新策略 epoch”。 +它不表示所有旧连接已排空;单独暴露旧 epoch 的活动连接数。 +默认允许健康存量 RPC 完成,之后将连接退休并阻止新 RPC 复用。 +对于持续复用或长连接,必须在请求获取 socket 时检查退休位,不能只在握手时检查。 +可选强制模式使旧连接失败,但不自动重放其中请求;其影响须由调用方显式选择。 +重新切回 AUTO 后通过 PROBING 恢复,不瞬间全量重建 RDMA 连接。 + +SocketMap / 连接池的复用键需区分路由模式、端口资源域和 policy epoch。 +同一逻辑 Channel 下,失败的 RDMA socket、临时 TCP socket 和探测 socket 不得误复用。 +TCP fallback 必须创建明确的 TCP transport/socket,不再进入 RDMA 初始化和握手。 +现有 socket 复活路径重新获取 lease,不把旧 endpoint 的路由状态带到新 generation。 +本地 RDMA 故障不得直接把仍可经 TCP 访问的服务实例长期标记为不可用; +需要验证 SetFailed 与健康检查、负载均衡惩罚的现有交互,保持单次 RPC 的错误通知。 + +## 多端口主备与注册内存 + +候选路径启动时配置并验证;顺序选择健康主路径,再选择健康备用路径。 +不在单个 QP 上修改活动路径,也不把旧 QP 搬到另一 HCA。 +切换意味着建立新 QP,重新交换正确的 GID/QPN 等信息。 +本地备用端口不一定能访问同一个对端,探测失败应隔离该对端路径。 + +当前全局 context / PD / MR / lkey 模型不能直接支撑跨 HCA 切换。 +跨 HCA 必须先引入 `RdmaDeviceContext`:每个 context 拥有 PD、事件源及 MR 注册表, +IOBuf 内存块按目标 PD 查找 lkey,不能把主设备的 lkey 用于备用设备。 +预注册备用设备需要容量预算及失败回滚;用户注册内存也需按设备注册和保活。 +不能安全注册到备用 PD 的 buffer 走明确的受控复制路径或 TCP,不能使用无效 lkey。 +同 HCA 不同端口若共享 PD 仍需验证 GID、路由和对端可达性;不默认等同于跨 HCA。 + +## SQ 停顿和 RNR 策略 + +SQ 窗口耗尽保留 EAGAIN。可选 watchdog 只跟踪有 outstanding WR 或待发送且被窗口阻塞的连接。 +使用单调时间记录最后发送完成及 credit 前进,不以 idle 时长判断故障。 +待发送数据、未完成 WR 和 credit 的状态分别记录,避免接收数据持续到达掩盖发送停顿。 +超过无进展阈值后使该连接失败并隔离对端路径;不导致整个端口隔离或进程退出。 +RPC 超时与传输 watchdog 独立,后者不延长任何调用的 deadline。 + +ACK timeout 指数、retry count 和 RNR retry 保持现有值作为默认,分别校验 verbs 范围。 +首版不提供无限 RNR 重试:它会把错误变成无法及时检测的停顿。 +允许的有限 RNR 重试需结合收包 buffer 补充延迟测试;调参不能替代滑动窗口正确性。 +watchdog 默认关闭,硬件和拥塞测试确定阈值后再建议生产开启。 + +## 提议配置与可观测性 + +以下名称尚未实现。配置启动时校验,运行时只允许通过专用 API 更新恢复模式。 + +| 参数 | 初始建议 | 约束 | +| --- | --- | --- | +| rdma_recovery_mode | legacy | legacy / auto / tcp_only;auto 显式启用 | +| rdma_path_cooldown_ms | 5000 | 正数,使用单调时钟 | +| rdma_path_max_cooldown_ms | 60000 | 不小于基础冷却时间 | +| rdma_recovery_probe_timeout_ms | 1000 | 正数;探测调用仍遵守更短的原 deadline | +| rdma_send_stall_timeout_ms | 0 | 0 关闭;启用时必须为正数 | +| rdma_backup_paths | 空 | 启动时配置;仅在设备资源域改造后支持 | + +暴露连接失败原因、RDMA-to-TCP 新建链次数、隔离次数、探测成功/失败、 +端口状态、旧 epoch 活动连接数、无进展发送时长及 SQ/RQ credit。 +指标避免使用无界远端地址作为 bvar 名称;对端细节放入有界诊断视图。 +日志包含 socket generation、device、port、WC status/vendor_err、errno 和策略 epoch, +对相同故障做限频,并保留首次根因,避免被 WR_FLUSH 覆盖。 + +## 并发、回收与停机 + +控制任务只持有弱 socket 标识和带代际的 route lease;执行时重新寻址并校验 generation。 +状态发布使用明确的 acquire/release 配对;事件回调不在持有策略锁时调用 SetFailed。 +防止 SetFailed、复活、探测定时器和端口事件之间形成锁顺序循环。 +故障后先停止准入和 WR 提交,再停止对应 CQ 处理并按现有安全流程 reset/drain QP/CQ, +最后释放发送 buffer、MR、PD 和 context。完成状态未知的 DMA 内存不能提前复用。 +停机取消探测和 watchdog、等待回调退出,再销毁设备资源。 +DEVICE_FATAL 的资源回收必须单独验证;无法证明安全排空时不移除当前 FATAL 保护。 + +## 分阶段交付 + +1. 单设备策略、错误分类、端口事件、冷却/探测、TCP 新连接恢复及受控运行时模式。 + 包含 SocketMap/连接池退休与复用边界、代际隔离和基本指标。 +2. 有界发送无进展检测、有限 RNR 参数校验与故障注入;默认保持现有背压行为。 +3. 多设备资源域、按 PD 注册内存及候选路径选择,完成后开放多端口/跨 HCA 主备。 + +每阶段单独实现和验证;阶段一不宣称支持跨 HCA 或 DEVICE_FATAL 无损恢复。 +本设计 PR 只增加文档;后续实现 PR 才启用功能和增加对应测试。 + +## 验证和验收标准 + +| 层级 | 场景 | 验收条件 | +| --- | --- | --- | +| 策略单测,注入时钟 | 冷却、退避、epoch 变化、并发准入 | 冷却零 RDMA 准入;到期最多一个探测;旧结果无效 | +| 连接测试,mock verbs | PORT_ERR 与并发建链、复活竞争 | 受影响连接失败并唤醒等待者;后续连接使用 TCP | +| 错误分类 | RNR、RETRY_EXC、WR_FLUSH、post_send errno | RNR 不隔离本地端口;连锁 flush 不重复计数 | +| 背压测试 | SQ/RQ 为零后 CQ/credit 恢复 | EAGAIN 可恢复,无误降级、忙循环或丢失唤醒 | +| watchdog | idle、持续接收但发送停顿、慢恢复 | idle 不触发;真实发送停顿有界失败;恢复更新进度 | +| RPC 语义 | 部分 post 后失败、服务端已执行 | 不在 transport 中重放;deadline 和重试配额不增加 | +| 复用与开关 | TCP_ONLY、AUTO、旧池连接、持续流量 | 新请求不复用退休 RDMA;TCP 成功不算 RDMA 探测成功 | +| 兼容性 | 开关关闭、新旧对端组合 | legacy 行为及现有握手格式不变 | +| 生命周期 | reset/revive、晚到 CQ 回调、停机 | 无 UAF、重复回收、提前释放 DMA buffer 或停机挂起 | +| 双端 Linux RDMA | 拔链路、恢复链路、接收资源不足 | 后续连接可走 TCP;恢复仅经探测;无重连风暴 | +| 双 HCA Linux RDMA | 主路径故障、不同 PD 的 lkey、注册失败 | 备用 QP 使用匹配 MR;失败可退 TCP;无无效 DMA | + +mock 验证不能代替真机验证。硬件验收记录 RDMA provider、固件、拓扑、 +RPC 成功率及恢复时延分布;使用受控 RoCE/IB 测试环境完成端口故障注入。 +分别测量关闭特性、健康 AUTO、冷却 TCP 三种情况下的吞吐、P99 和 CPU。 +合入实现前需给出恢复预算及性能回归容限;本草案不提供未经测量的性能承诺。 + +## 待评审决策 + +- 首阶段运行时控制是否采用进程级 API,后续是否需要 Channel 级覆盖? +- 长连接退休的默认排空期限和强制失败入口如何与现有连接池接口对齐? +- 多 HCA 用户内存注册是否提供显式 registration set,还是通过逐设备查询接口? +- 发送停顿阈值及允许的有限 RNR 重试是否需要按部署配置档提供建议? From 88ece2db9bf84e4bda79070bd4d90e0a6a84b2d0 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:28:05 -0700 Subject: [PATCH 02/29] Simplify RDMA recovery design --- docs/cn/rdma.md | 4 +- docs/cn/rdma_runtime_recovery.md | 271 +++++++++---------------------- 2 files changed, 83 insertions(+), 192 deletions(-) diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index 96fa00a103..052fbc13d0 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,7 +1,7 @@ # 运行期故障恢复设计 -[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md) 为尚未实现的设计草案, -讨论端口故障隔离、冷却与恢复探测、运行时降级控制及多端口主备。 +端口故障隔离、冷却探测、运行时降级及多端口主备方案见 +[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md)。 # 编译 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index 8e1af1ef47..d9621406c4 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -19,213 +19,104 @@ under the License. # RDMA 运行期故障恢复与 TCP 降级设计 -状态:设计草案,尚未实现。本文中的接口、参数和指标均为提案。 +设计提案,基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 -基线为 LinQuickDev/brpc master 的 -`87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 -本提案改变后续连接的传输选择;不承诺存量 RPC 无损迁移或 exactly-once。 +## 当前问题 -## 问题与源码依据 +RDMA 握手阶段支持回落 TCP,建链后的错误 CQE 主要通过 `Socket::SetFailed` 处理。 +`PORT_ERR` / `PORT_ACTIVE` 事件仅做确认,缺少端口隔离、冷却和恢复探测。 +`GlobalDisableRdma` 使用 FATAL,需要独立的运行时降级入口。 -当前握手可以回落 TCP,但 RDMA 建立后的故障主要通过连接失败处理: +相关代码:`src/brpc/rdma/rdma_endpoint.cpp`、`src/brpc/rdma/rdma_helper.cpp`、 +`src/brpc/rdma_transport.cpp`。 -| 位置 | 当前行为 | 需要补充的能力 | -| --- | --- | --- | -| `RdmaConnect::StartConnect`、握手状态机 | RDMA 不可用或协商失败时回落 TCP | 后续建链能记住运行期故障并绕过故障路径 | -| `RdmaEndpoint::PollCq` | 错误 CQE 调用 `Socket::SetFailed` | 错误分类、隔离和恢复探测 | -| `OnRdmaAsyncEvent` | PORT_ERR / PORT_ACTIVE 只确认事件 | 端口状态与连接准入联动 | -| `GlobalDisableRdma` | 修改可用状态并记录 FATAL | 独立、非致命的运维降级控制 | -| `CutFromIOBufList` | 窗口耗尽返回 EAGAIN;post_send 失败按不可恢复处理 | 保留背压语义,检测有未完成发送时的长期无进展 | -| `BringUpQp` | timeout=14、retry_cnt=7、rnr_retry=0 | 明确错误语义及受限配置,而非把所有错误归为超时 | +## 恢复流程 -源码分别位于 `src/brpc/rdma/rdma_endpoint.cpp`、 -`src/brpc/rdma/rdma_helper.cpp` 和 `src/brpc/rdma_transport.cpp`。 -RPC deadline、verbs ACK 重试耗尽、RNR 重试耗尽和本地 SQ 背压是不同事件。 +故障检测 → 隔离故障路径 → 新连接选择备用 RDMA 或 TCP → 冷却 → 单连接探测 → 恢复。 -## 目标和兼容性 +新增 `RdmaRecoveryPolicy`,分别管理本地端口和对端路径的健康状态。 +本地端口以 device、port 标识,对端路径增加远端地址。 +`RdmaTransport` 建链时获取路由快照,`RdmaEndpoint` 持有对应设备资源。 -1. 本地 RDMA 端口故障后,快速停止该端口的新建链并使相关连接失败。 -2. 后续连接优先使用可达的备用 RDMA 路径,无可用路径时通过新 TCP 连接恢复。 -3. 对故障路径设置冷却和有界恢复探测,避免每次 RPC 都重新尝试故障 RDMA。 -4. 提供运行时 TCP-only 开关,并明确存量连接的处理方式。 -5. 给出 SQ 停顿、RNR 和链路错误的独立策略、指标和测试标准。 +发生故障时,先发布隔离状态,再使受影响连接失败并唤醒写等待者。 +后续连接按主 RDMA、备用 RDMA、TCP 的顺序选择可用路径。 +TCP 降级使用新连接;存量 RPC 交给现有 Controller 重试策略处理,沿用原配额和 deadline。 -自动恢复默认关闭;启用前保留现有单设备、单端口和握手协议行为。 -TCP fallback 遵循现有协议与 SSL 限制,不绕过认证,不隐式降低安全配置。 -发布第一阶段只实现单设备故障隔离和 TCP 恢复;跨 HCA 主备在资源模型改造后提供。 +## 状态与错误处理 -## 恢复边界 +路径状态为 HEALTHY、COOLDOWN、PROBING、DISABLED。 -故障连接执行正常 SetFailed、唤醒写等待者并进入回收流程。 -策略先发布故障状态,再传播连接失败,使同时发生的新建链看到最新路由。 -已经建立的 RDMA 连接不改为在原 TCP fd 上发送 RPC 数据。 -该 fd 在握手后有独立语义,直接切换会造成消息边界、解析状态和顺序问题。 +| 事件 | 处理 | +| --- | --- | +| PORT_ERR | 端口进入 COOLDOWN,关闭该端口上的连接 | +| PORT_ACTIVE | 标记端口可探测,冷却结束后进入 PROBING | +| QP/CQ 错误、RETRY_EXC | 连接失败并隔离对应对端路径 | +| RNR_RETRY_EXC | 隔离对端路径,记录接收资源或流控异常 | +| SQ/RQ 窗口耗尽 | 返回 EAGAIN,等待 completion 或 credit | +| post_send 失败 | 保存 errno,使连接失败并按错误类型更新路径状态 | +| DEVICE_FATAL | 禁用对应设备资源域,沿用现有 FATAL 处理 | -只由现有 Controller、RetryPolicy、max_retry 和 RPC deadline 决定是否重试。 -新增策略不增加重试配额、不重置 deadline、不自动重放已 post 的 WR。 -请求可能已被服务端执行时保留现有错误语义;有副作用的业务仍需幂等键或去重。 -恢复策略只保证后续连接避开故障 RDMA,不保证正在执行的调用成功。 +冷却使用单调时钟,连续失败采用指数退避和 jitter。 +到期后最多准入一个 RDMA 探测连接,其余新连接继续走备用路径或 TCP。 +探测使用正常业务流量;收到有效 RDMA 数据且发送获得完成后恢复 HEALTHY, +失败或探测超时则重新冷却。 -## 组件与资源所有权 +每次状态更新携带 generation 和 policy epoch。 +同一连接代际只报告一次故障,晚到事件按代际过滤。 -在 `src/brpc/rdma/` 新增 `RdmaRecoveryPolicy`,管理准入、冷却和探测。 -`RdmaTransport` 在建链前获取一次不可变的 route lease; -`RdmaEndpoint` 的 QP、CQ、GID、port 和注册内存绑定到该 lease。 +## 运行时降级 -状态分为两个作用域: +提供进程级 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 和 setter/getter。 +LEGACY 为默认值;AUTO 启用恢复策略;TCP_ONLY 将后续请求切到 TCP。 -- 本地端口键:device identity、port number、资源 generation。 -- 对端路径键:本地端口键、远端地址、传输模式、协商相关身份。 +模式更新递增 policy epoch。TCP_ONLY 下,健康存量 RPC 完成后连接退休; +请求获取 socket 时检查退休状态,新请求使用新连接。 +切回 AUTO 后进入 PROBING,逐步恢复 RDMA。 -QP 错误先影响连接或对端路径,不根据一个对端失败就禁用整块网卡。 -新连接可读取端口状态和对端状态的不可变快照;数据发送热路径不加全局锁。 -有界缓存使用过期时间和容量限制;只逐出无活动 lease、无探测的对端状态。 -端口配置固定在启动时,运维开关在运行时更新策略 epoch。 +SocketMap 和连接池按传输模式、设备资源域和 epoch 区分连接。 +socket 复活时重新获取路由快照。服务实例的健康检查与 RDMA 路径状态分开维护。 -## 状态机与故障分类 +## 多端口主备 -端口和对端路径分别维护 HEALTHY、COOLDOWN、PROBING、DISABLED。 -单调时钟驱动冷却;每个状态更新带 generation 和 policy epoch。 +启动时配置主备路径,切换时创建新 QP 并交换对应 GID/QPN。 +同 HCA 多端口共享 PD 时复用注册内存,各端口独立维护路由和健康状态。 -| 输入 | 状态变化 | 行为 | -| --- | --- | --- | -| 本地 PORT_ERR | 端口进入 COOLDOWN | 禁止新 RDMA;使绑定该 generation 的连接失败 | -| PORT_ACTIVE | 保留隔离,记录链路恢复 | 不立即恢复全量流量,等待冷却和探测 | -| QP/CQ 错误 | 连接失败,必要时对端路径进入 COOLDOWN | 不据此断定本地端口故障 | -| RETRY_EXC | 对端路径进入 COOLDOWN | 原因可能是路径、对端或网络;不累计为本地端口故障 | -| RNR_RETRY_EXC | 对端路径进入 COOLDOWN | 记录接收资源/流控异常,不切坏整个本地端口 | -| 正常 SQ/RQ 窗口为零 | 不改变健康状态 | EAGAIN 背压,等待 completion / credit | -| post_send 失败 | 连接失败,按 errno 分类 | 保存原始错误;不直接重投 WR 或循环忙重试 | -| 运维 TCP-only | 准入进入 DISABLED | 新建链直接选择 TCP,按下文处理存量连接 | -| DEVICE_FATAL | 对应资源域永久 DISABLED | 仅在安全排空得到验证后支持非致命降级,否则保留现有 FATAL | - -禁止把 CQ flush 连锁错误重复计为多次独立故障。 -同一 socket generation 的失败使用一次性报告,过期事件不得修改新一代状态。 -内存注册错误、远端访问错误和本地保护错误保留独立诊断;不以自动重试掩盖编程错误。 - -## 冷却与恢复探测 - -首次隔离使用基础冷却时间,连续探测失败指数退避至上限,并加入有界 jitter。 -冷却期所有新连接跳过该路径,选择健康候选路径或 TCP。 -冷却结束不等于恢复:原子准入最多一个新 RDMA 连接作为探测,其余请求继续走 TCP。 -PORT_ERR 后还必须先确认本地端口 ACTIVE,才能发放探测 lease。 - -握手成功只能证明建链成功。探测连接收到有效 RDMA 数据、且至少一次发送获得完成后, -才把路径恢复为 HEALTHY;不能以 TCP 成功证明 RDMA 已恢复。 -不构造会产生业务副作用的探测 RPC;使用正常业务流量并受原调用 deadline 约束。 -探测在预算内失败或没有足够证据时重新冷却,不长期占用探测名额。 -一条探测连接同时持有端口和对端 lease,成功/失败只释放对应 epoch 的名额。 - -## 运行时控制与连接复用 - -提议 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 及进程级原子 setter/getter。 -setter 校验输入、增加 epoch,并唤醒控制任务;不直接销毁 QP 或 MR。 -运维系统可调用 API,初版不新增可远程修改状态的 HTTP 接口。 - -TCP_ONLY 的完成语义是“停止新 RDMA 准入并返回新策略 epoch”。 -它不表示所有旧连接已排空;单独暴露旧 epoch 的活动连接数。 -默认允许健康存量 RPC 完成,之后将连接退休并阻止新 RPC 复用。 -对于持续复用或长连接,必须在请求获取 socket 时检查退休位,不能只在握手时检查。 -可选强制模式使旧连接失败,但不自动重放其中请求;其影响须由调用方显式选择。 -重新切回 AUTO 后通过 PROBING 恢复,不瞬间全量重建 RDMA 连接。 - -SocketMap / 连接池的复用键需区分路由模式、端口资源域和 policy epoch。 -同一逻辑 Channel 下,失败的 RDMA socket、临时 TCP socket 和探测 socket 不得误复用。 -TCP fallback 必须创建明确的 TCP transport/socket,不再进入 RDMA 初始化和握手。 -现有 socket 复活路径重新获取 lease,不把旧 endpoint 的路由状态带到新 generation。 -本地 RDMA 故障不得直接把仍可经 TCP 访问的服务实例长期标记为不可用; -需要验证 SetFailed 与健康检查、负载均衡惩罚的现有交互,保持单次 RPC 的错误通知。 - -## 多端口主备与注册内存 - -候选路径启动时配置并验证;顺序选择健康主路径,再选择健康备用路径。 -不在单个 QP 上修改活动路径,也不把旧 QP 搬到另一 HCA。 -切换意味着建立新 QP,重新交换正确的 GID/QPN 等信息。 -本地备用端口不一定能访问同一个对端,探测失败应隔离该对端路径。 - -当前全局 context / PD / MR / lkey 模型不能直接支撑跨 HCA 切换。 -跨 HCA 必须先引入 `RdmaDeviceContext`:每个 context 拥有 PD、事件源及 MR 注册表, -IOBuf 内存块按目标 PD 查找 lkey,不能把主设备的 lkey 用于备用设备。 -预注册备用设备需要容量预算及失败回滚;用户注册内存也需按设备注册和保活。 -不能安全注册到备用 PD 的 buffer 走明确的受控复制路径或 TCP,不能使用无效 lkey。 -同 HCA 不同端口若共享 PD 仍需验证 GID、路由和对端可达性;不默认等同于跨 HCA。 - -## SQ 停顿和 RNR 策略 - -SQ 窗口耗尽保留 EAGAIN。可选 watchdog 只跟踪有 outstanding WR 或待发送且被窗口阻塞的连接。 -使用单调时间记录最后发送完成及 credit 前进,不以 idle 时长判断故障。 -待发送数据、未完成 WR 和 credit 的状态分别记录,避免接收数据持续到达掩盖发送停顿。 -超过无进展阈值后使该连接失败并隔离对端路径;不导致整个端口隔离或进程退出。 -RPC 超时与传输 watchdog 独立,后者不延长任何调用的 deadline。 - -ACK timeout 指数、retry count 和 RNR retry 保持现有值作为默认,分别校验 verbs 范围。 -首版不提供无限 RNR 重试:它会把错误变成无法及时检测的停顿。 -允许的有限 RNR 重试需结合收包 buffer 补充延迟测试;调参不能替代滑动窗口正确性。 -watchdog 默认关闭,硬件和拥塞测试确定阈值后再建议生产开启。 - -## 提议配置与可观测性 - -以下名称尚未实现。配置启动时校验,运行时只允许通过专用 API 更新恢复模式。 - -| 参数 | 初始建议 | 约束 | -| --- | --- | --- | -| rdma_recovery_mode | legacy | legacy / auto / tcp_only;auto 显式启用 | -| rdma_path_cooldown_ms | 5000 | 正数,使用单调时钟 | -| rdma_path_max_cooldown_ms | 60000 | 不小于基础冷却时间 | -| rdma_recovery_probe_timeout_ms | 1000 | 正数;探测调用仍遵守更短的原 deadline | -| rdma_send_stall_timeout_ms | 0 | 0 关闭;启用时必须为正数 | -| rdma_backup_paths | 空 | 启动时配置;仅在设备资源域改造后支持 | - -暴露连接失败原因、RDMA-to-TCP 新建链次数、隔离次数、探测成功/失败、 -端口状态、旧 epoch 活动连接数、无进展发送时长及 SQ/RQ credit。 -指标避免使用无界远端地址作为 bvar 名称;对端细节放入有界诊断视图。 -日志包含 socket generation、device、port、WC status/vendor_err、errno 和策略 epoch, -对相同故障做限频,并保留首次根因,避免被 WR_FLUSH 覆盖。 - -## 并发、回收与停机 - -控制任务只持有弱 socket 标识和带代际的 route lease;执行时重新寻址并校验 generation。 -状态发布使用明确的 acquire/release 配对;事件回调不在持有策略锁时调用 SetFailed。 -防止 SetFailed、复活、探测定时器和端口事件之间形成锁顺序循环。 -故障后先停止准入和 WR 提交,再停止对应 CQ 处理并按现有安全流程 reset/drain QP/CQ, -最后释放发送 buffer、MR、PD 和 context。完成状态未知的 DMA 内存不能提前复用。 -停机取消探测和 watchdog、等待回调退出,再销毁设备资源。 -DEVICE_FATAL 的资源回收必须单独验证;无法证明安全排空时不移除当前 FATAL 保护。 - -## 分阶段交付 - -1. 单设备策略、错误分类、端口事件、冷却/探测、TCP 新连接恢复及受控运行时模式。 - 包含 SocketMap/连接池退休与复用边界、代际隔离和基本指标。 -2. 有界发送无进展检测、有限 RNR 参数校验与故障注入;默认保持现有背压行为。 -3. 多设备资源域、按 PD 注册内存及候选路径选择,完成后开放多端口/跨 HCA 主备。 - -每阶段单独实现和验证;阶段一不宣称支持跨 HCA 或 DEVICE_FATAL 无损恢复。 -本设计 PR 只增加文档;后续实现 PR 才启用功能和增加对应测试。 - -## 验证和验收标准 - -| 层级 | 场景 | 验收条件 | +跨 HCA 引入 `RdmaDeviceContext`,分别持有 context、PD、事件源和 MR 注册表。 +IOBuf 和用户内存按目标 PD 查找 lkey,并管理各设备上的注册与引用生命周期。 +备用设备注册失败时选择 TCP。 + +回收顺序为:停止准入和 WR 提交,停止 CQ 处理,reset/drain QP/CQ, +再释放 buffer、MR、PD 和 context。停机先取消探测与 watchdog,再等待回调退出。 + +## SQ 停顿与 RNR + +正常窗口耗尽保持 EAGAIN 背压。发送 watchdog 跟踪 outstanding WR、 +待发送数据和 credit 的进展;超过阈值后使连接失败并隔离对端路径。 +计时从发送受阻开始,发送完成或 credit 前进时更新。 + +ACK timeout、retry count 和 RNR retry 保持当前默认值,支持配置有限重试次数。 +RNR 重试耗尽作为对端路径故障处理,发送停顿检测独立于 RPC deadline。 + +## 配置与指标 + +| 参数 | 建议默认值 | 用途 | | --- | --- | --- | -| 策略单测,注入时钟 | 冷却、退避、epoch 变化、并发准入 | 冷却零 RDMA 准入;到期最多一个探测;旧结果无效 | -| 连接测试,mock verbs | PORT_ERR 与并发建链、复活竞争 | 受影响连接失败并唤醒等待者;后续连接使用 TCP | -| 错误分类 | RNR、RETRY_EXC、WR_FLUSH、post_send errno | RNR 不隔离本地端口;连锁 flush 不重复计数 | -| 背压测试 | SQ/RQ 为零后 CQ/credit 恢复 | EAGAIN 可恢复,无误降级、忙循环或丢失唤醒 | -| watchdog | idle、持续接收但发送停顿、慢恢复 | idle 不触发;真实发送停顿有界失败;恢复更新进度 | -| RPC 语义 | 部分 post 后失败、服务端已执行 | 不在 transport 中重放;deadline 和重试配额不增加 | -| 复用与开关 | TCP_ONLY、AUTO、旧池连接、持续流量 | 新请求不复用退休 RDMA;TCP 成功不算 RDMA 探测成功 | -| 兼容性 | 开关关闭、新旧对端组合 | legacy 行为及现有握手格式不变 | -| 生命周期 | reset/revive、晚到 CQ 回调、停机 | 无 UAF、重复回收、提前释放 DMA buffer 或停机挂起 | -| 双端 Linux RDMA | 拔链路、恢复链路、接收资源不足 | 后续连接可走 TCP;恢复仅经探测;无重连风暴 | -| 双 HCA Linux RDMA | 主路径故障、不同 PD 的 lkey、注册失败 | 备用 QP 使用匹配 MR;失败可退 TCP;无无效 DMA | - -mock 验证不能代替真机验证。硬件验收记录 RDMA provider、固件、拓扑、 -RPC 成功率及恢复时延分布;使用受控 RoCE/IB 测试环境完成端口故障注入。 -分别测量关闭特性、健康 AUTO、冷却 TCP 三种情况下的吞吐、P99 和 CPU。 -合入实现前需给出恢复预算及性能回归容限;本草案不提供未经测量的性能承诺。 - -## 待评审决策 - -- 首阶段运行时控制是否采用进程级 API,后续是否需要 Channel 级覆盖? -- 长连接退休的默认排空期限和强制失败入口如何与现有连接池接口对齐? -- 多 HCA 用户内存注册是否提供显式 registration set,还是通过逐设备查询接口? -- 发送停顿阈值及允许的有限 RNR 重试是否需要按部署配置档提供建议? +| rdma_recovery_mode | legacy | 选择恢复模式 | +| rdma_path_cooldown_ms | 5000 | 初始冷却时间 | +| rdma_path_max_cooldown_ms | 60000 | 退避上限 | +| rdma_recovery_probe_timeout_ms | 1000 | 探测预算,受调用 deadline 约束 | +| rdma_send_stall_timeout_ms | 0 | 发送停顿阈值,0 关闭 | +| rdma_backup_paths | 空 | 启动时配置主备路径 | + +指标覆盖端口状态、隔离次数、TCP 降级次数、探测结果、旧 epoch 活动连接数和发送停顿时长。 +日志记录 device、port、socket generation、WC status/vendor_err 和 errno。 + +## 实现顺序与验证 + +1. 实现单设备故障分类、端口隔离、冷却探测、TCP 降级和运行时模式切换。 +2. 增加发送停顿检测、有限 RNR 重试配置和诊断指标。 +3. 拆分设备资源域,支持多端口及跨 HCA 主备。 + +单测覆盖冷却与并发探测、错误分类、窗口恢复、模式切换、连接复用和代际回收。 +双端 Linux RDMA 测试覆盖断链恢复、RNR 和发送停顿;双 HCA 测试验证主备切换与 MR 注册。 +对比 LEGACY、健康 AUTO 和降级 TCP 的恢复时延、吞吐、P99 和 CPU。 From 3846c94d373f71b5ecb1a02a1639166081059e34 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:42:31 -0700 Subject: [PATCH 03/29] Add initial RDMA runtime recovery and TCP fallback --- docs/cn/rdma_runtime_recovery.md | 28 ++++- src/brpc/rdma/rdma_endpoint.cpp | 17 ++- src/brpc/rdma/rdma_endpoint.h | 3 + src/brpc/rdma/rdma_helper.cpp | 12 +- src/brpc/rdma/rdma_helper.h | 3 + src/brpc/rdma/rdma_recovery_policy.cpp | 144 ++++++++++++++++++++++ src/brpc/rdma/rdma_recovery_policy.h | 67 ++++++++++ test/brpc_rdma_recovery_unittest.cpp | 156 +++++++++++++++++++++++ test/brpc_rdma_unittest.cpp | 163 +++++++++++++++++++++++++ 9 files changed, 588 insertions(+), 5 deletions(-) create mode 100644 src/brpc/rdma/rdma_recovery_policy.cpp create mode 100644 src/brpc/rdma/rdma_recovery_policy.h create mode 100644 test/brpc_rdma_recovery_unittest.cpp diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index d9621406c4..058a2088f6 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -19,7 +19,33 @@ under the License. # RDMA 运行期故障恢复与 TCP 降级设计 -设计提案,基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 +基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 + +## 首版实现 + +`RdmaRecoveryPolicy` 已接入客户端和服务端握手入口,控制新连接的 RDMA 准入: + +- LEGACY 为默认模式,沿用现有行为;TCP_ONLY 让新连接使用 TCP。 +- AUTO 下,客户端收到 RETRY_EXC / RNR_RETRY_EXC 后,按远端地址冷却 5 秒。 +- 当前设备的 PORT_ERR 阻止新 RDMA 建链,端口恢复且冷却结束后重新准入。 +- 对端冷却缓存最多 1024 项,满时淘汰最早到期项;模式切换用 epoch 过滤旧连接错误。 + +通过进程级 API 配置: + +```cpp +#include "brpc/rdma/rdma_recovery_policy.h" + +brpc::rdma::SetRdmaRecoveryCooldownMs(5000); +brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::AUTO); +// 运维降级 +brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::TCP_ONLY); +``` + +当前模式切换作用于新建连接,存量连接继续使用已协商的传输。 +冷却到期后重新开放准入,TCP 连接沿用现有复用与重建机制。 +后续迭代增加存量连接退休、单连接探测、指数退避、发送 watchdog 和多端口主备。 + +以下为完整方案。 ## 当前问题 diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 0dc41c9dd1..33eebaa08c 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -29,6 +29,7 @@ #include "brpc/reloadable_flags.h" #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma_transport.h" #include "brpc/rdma/rdma_handshake.h" @@ -131,6 +132,7 @@ RdmaEndpoint::RdmaEndpoint(Socket* s) : _socket(s) , _state(UNINIT) , _handshake_version(0) + , _recovery_epoch(0) , _resource(nullptr) , _send_cq_events(0) , _recv_cq_events(0) @@ -178,6 +180,7 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; + _recovery_epoch = 0; _outgoing_ece.reset(); _resource = nullptr; _send_cq_events = 0; @@ -212,7 +215,10 @@ void RdmaConnect::StartConnect(const Socket* socket, if (Socket::Address(socket->id(), &s) != 0) { return; } - if (!IsRdmaAvailable()) { + if (!IsRdmaAvailable() || + !GlobalRdmaRecoveryPolicy().AllowNewConnection( + butil::endpoint2str(socket->remote_side()).c_str(), + &rdma_transport->_rdma_ep->_recovery_epoch)) { rdma_transport->_rdma_state = RdmaTransport::RDMA_OFF; rdma_transport->_rdma_ep->_state.store( RdmaEndpoint::FALLBACK_TCP, butil::memory_order_release); @@ -683,7 +689,9 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s } // Negotiate + allocate resources. - bool negotiated = r == RemoteHelloResult::NEGOTIATED; + bool negotiated = r == RemoteHelloResult::NEGOTIATED && + GlobalRdmaRecoveryPolicy().AllowNewConnection( + "", &ep->_recovery_epoch); if (negotiated) { ep->ApplyRemoteHello(remote); ep->_state.store(S_ALLOC_QPCQ, butil::memory_order_relaxed); @@ -1812,6 +1820,11 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { + if (s->CreatedByConnect() && IsRdmaPathFailure(wc[i].status)) { + GlobalRdmaRecoveryPolicy().ReportPeerFailure( + butil::endpoint2str(s->remote_side()).c_str(), + ep->_recovery_epoch); + } PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); s->SetFailed(ERDMA, "RDMA completion error(%d) from %s: %s", diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index eda5e558ef..eff186be50 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -307,6 +307,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // 3 = v3 "RDM3" int _handshake_version; + // Admission epoch used to discard failures from an earlier recovery mode. + uint64_t _recovery_epoch; + // ECE payload to advertise in the next local hello: // Client: the locally queried ECE capabilities (filled // before C_HELLO_SEND); diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index 7e671a85ba..f0ff394a37 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -30,6 +30,7 @@ #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" namespace butil { @@ -313,8 +314,10 @@ static void OnRdmaAsyncEvent(Socket* m) { } case IBV_EVENT_PORT_ACTIVE: case IBV_EVENT_PORT_ERR: { - // Port up/down will lead these two events. - // The port error is recoverable. + if (event.element.port_num == g_port_num) { + GlobalRdmaRecoveryPolicy().ReportPortState( + event.event_type == IBV_EVENT_PORT_ACTIVE); + } IbvAckAsyncEvent(&event); break; } @@ -699,6 +702,11 @@ bool IsRdmaAvailable() { return g_rdma_available.load(butil::memory_order_acquire); } +bool IsRdmaPathFailure(ibv_wc_status status) { + return status == IBV_WC_RETRY_EXC_ERR || + status == IBV_WC_RNR_RETRY_EXC_ERR; +} + void GlobalDisableRdma() { if (g_rdma_available.exchange(false, butil::memory_order_acquire)) { LOG(FATAL) << "RDMA is disabled due to some unrecoverable problem"; diff --git a/src/brpc/rdma/rdma_helper.h b/src/brpc/rdma/rdma_helper.h index 2383d2d8b4..d3a840cf1d 100644 --- a/src/brpc/rdma/rdma_helper.h +++ b/src/brpc/rdma/rdma_helper.h @@ -89,6 +89,9 @@ bool IsRdmaAvailable(); // Disable RDMA in the remaining lifetime of the process void GlobalDisableRdma(); +// Completion failures that indicate an unusable path or exhausted receive credits. +bool IsRdmaPathFailure(ibv_wc_status status); + // If the given protocol supported by RDMA bool SupportedByRdma(std::string protocol); diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp new file mode 100644 index 0000000000..801605cb90 --- /dev/null +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/rdma/rdma_recovery_policy.h" +#include "butil/time.h" + +namespace brpc { +namespace rdma { + +RdmaRecoveryPolicy::RdmaRecoveryPolicy(Clock clock, size_t max_peers) + : _clock(clock) + , _max_peers(max_peers) + , _mode(RdmaRecoveryMode::LEGACY) + , _epoch(1) + , _cooldown_us(5000000) + , _port_active(true) + , _port_retry_at(0) { +} + +bool RdmaRecoveryPolicy::SetMode(RdmaRecoveryMode mode) { + if (mode != RdmaRecoveryMode::LEGACY && mode != RdmaRecoveryMode::AUTO && + mode != RdmaRecoveryMode::TCP_ONLY) { + return false; + } + std::lock_guard lock(_mutex); + if (_mode != mode) { + _mode = mode; + ++_epoch; + _peers.clear(); + } + return true; +} + +RdmaRecoveryMode RdmaRecoveryPolicy::GetMode() { + std::lock_guard lock(_mutex); + return _mode; +} + +bool RdmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { + if (cooldown_ms <= 0 || cooldown_ms > INT32_MAX) { + return false; + } + std::lock_guard lock(_mutex); + _cooldown_us = cooldown_ms * 1000; + return true; +} + +bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, + uint64_t* epoch) { + std::lock_guard lock(_mutex); + if (epoch != nullptr) { + *epoch = _epoch; + } + if (_mode == RdmaRecoveryMode::LEGACY) { + return true; + } + if (_mode == RdmaRecoveryMode::TCP_ONLY) { + return false; + } + const int64_t now = _clock(); + if (!_port_active || now < _port_retry_at) { + return false; + } + auto it = _peers.find(peer); + if (it == _peers.end()) { + return true; + } + if (now < it->second) { + return false; + } + _peers.erase(it); + return true; +} + +void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, + uint64_t epoch) { + std::lock_guard lock(_mutex); + if (_mode != RdmaRecoveryMode::AUTO || epoch != _epoch || + peer.empty() || _max_peers == 0) { + return; + } + const int64_t now = _clock(); + for (auto it = _peers.begin(); it != _peers.end();) { + if (it->second <= now) { + it = _peers.erase(it); + } else { + ++it; + } + } + if (_peers.count(peer) == 0 && _peers.size() == _max_peers) { + auto oldest = _peers.begin(); + for (auto it = _peers.begin(); it != _peers.end(); ++it) { + if (it->second < oldest->second) { + oldest = it; + } + } + _peers.erase(oldest); + } + _peers[peer] = now + _cooldown_us; +} + +void RdmaRecoveryPolicy::ReportPortState(bool active) { + std::lock_guard lock(_mutex); + _port_active = active; + if (!active) { + _port_retry_at = _clock() + _cooldown_us; + } +} + +RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { + // Async event callbacks may outlive ordinary static destructors. + static RdmaRecoveryPolicy* policy = + new RdmaRecoveryPolicy(butil::monotonic_time_us); + return *policy; +} + +bool SetRdmaRecoveryMode(RdmaRecoveryMode mode) { + return GlobalRdmaRecoveryPolicy().SetMode(mode); +} + +RdmaRecoveryMode GetRdmaRecoveryMode() { + return GlobalRdmaRecoveryPolicy().GetMode(); +} + +bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms) { + return GlobalRdmaRecoveryPolicy().SetCooldownMs(cooldown_ms); +} + +} // namespace rdma +} // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h new file mode 100644 index 0000000000..d5f430d6db --- /dev/null +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_RDMA_RECOVERY_POLICY_H +#define BRPC_RDMA_RECOVERY_POLICY_H + +#include +#include +#include +#include +#include + +namespace brpc { +namespace rdma { + +enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; + +// Admission policy for new connections on the process's selected RDMA port. +// An empty peer selects server admission, which only checks the local port. +class RdmaRecoveryPolicy { +public: + typedef int64_t (*Clock)(); + explicit RdmaRecoveryPolicy(Clock clock, size_t max_peers = 1024); + bool SetMode(RdmaRecoveryMode mode); + RdmaRecoveryMode GetMode(); + // Positive milliseconds, at most INT32_MAX. Applies to future failures. + bool SetCooldownMs(int64_t cooldown_ms); + bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); + void ReportPeerFailure(const std::string& peer, uint64_t epoch); + void ReportPortState(bool active); + +private: + std::mutex _mutex; + const Clock _clock; + const size_t _max_peers; + RdmaRecoveryMode _mode; + uint64_t _epoch; + int64_t _cooldown_us; + bool _port_active; + int64_t _port_retry_at; + std::map _peers; +}; + +// Process-lifetime policy, shared by client and server handshake admission. +RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy(); +bool SetRdmaRecoveryMode(RdmaRecoveryMode mode); +RdmaRecoveryMode GetRdmaRecoveryMode(); +bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms); + +} // namespace rdma +} // namespace brpc + +#endif // BRPC_RDMA_RECOVERY_POLICY_H diff --git a/test/brpc_rdma_recovery_unittest.cpp b/test/brpc_rdma_recovery_unittest.cpp new file mode 100644 index 0000000000..8470230e57 --- /dev/null +++ b/test/brpc_rdma_recovery_unittest.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include "brpc/rdma/rdma_recovery_policy.h" + +namespace brpc { +namespace rdma { +namespace { + +std::atomic now_us(1000000); +int64_t FakeClock() { return now_us.load(); } + +class RdmaRecoveryTest : public testing::Test { +protected: + RdmaRecoveryTest() : policy(FakeClock, 2) {} + void SetUp() override { now_us = 1000000; } + RdmaRecoveryPolicy policy; +}; + +TEST_F(RdmaRecoveryTest, legacy_preserves_admission) { + policy.ReportPortState(false); + uint64_t epoch; + EXPECT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.SetMode(RdmaRecoveryMode::AUTO)); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, peer_failure_only_cools_that_peer) { + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetCooldownMs(10); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("")); + now_us += 9999; + EXPECT_FALSE(policy.AllowNewConnection("a")); + ++now_us; + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, port_requires_active_and_elapsed_cooldown) { + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetCooldownMs(10); + policy.ReportPortState(false); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); + policy.ReportPortState(true); + EXPECT_FALSE(policy.AllowNewConnection("a")); + now_us += 10000; + EXPECT_TRUE(policy.AllowNewConnection("a")); + policy.ReportPortState(false); + now_us += 10000; + EXPECT_FALSE(policy.AllowNewConnection("a")); + policy.ReportPortState(true); + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t old_epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &old_epoch)); + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.ReportPeerFailure("a", old_epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + EXPECT_NE(old_epoch, epoch); + // Reapplying AUTO does not clear existing quarantine. + policy.ReportPeerFailure("a", epoch); + policy.SetMode(RdmaRecoveryMode::AUTO); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + ++now_us; + policy.ReportPeerFailure("b", epoch); + ++now_us; + policy.ReportPeerFailure("c", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_FALSE(policy.AllowNewConnection("c")); + now_us += 5000000; + policy.ReportPeerFailure("d", epoch); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("c")); + EXPECT_FALSE(policy.AllowNewConnection("d")); +} + +TEST_F(RdmaRecoveryTest, validates_configuration) { + EXPECT_FALSE(policy.SetMode(static_cast(-1))); + EXPECT_EQ(RdmaRecoveryMode::LEGACY, policy.GetMode()); + EXPECT_FALSE(policy.SetCooldownMs(0)); + EXPECT_FALSE(policy.SetCooldownMs(-1)); + EXPECT_FALSE(policy.SetCooldownMs(static_cast(INT32_MAX) + 1)); + EXPECT_TRUE(policy.SetCooldownMs(INT32_MAX)); + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { + std::vector workers; + for (int i = 0; i < 8; ++i) { + workers.emplace_back([this, i] { + for (int j = 0; j < 200; ++j) { + uint64_t epoch; + const std::string peer = std::to_string(i); + policy.SetMode(j % 2 ? RdmaRecoveryMode::AUTO : + RdmaRecoveryMode::TCP_ONLY); + policy.AllowNewConnection(peer, &epoch); + policy.ReportPeerFailure(peer, epoch); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); +} + +} // namespace +} // namespace rdma +} // namespace brpc diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index 6ec54ed90b..bd808ca0bc 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -35,6 +35,7 @@ #include "brpc/acceptor.h" #include "brpc/channel.h" #include "brpc/controller.h" +#include "brpc/input_messenger.h" #include "brpc/server.h" #include "brpc/socket.h" #include "brpc/errno.pb.h" @@ -47,6 +48,7 @@ #include "brpc/rdma/rdma_handshake_constants.h" #include "brpc/rdma/rdma_handshake.pb.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" #include "echo.pb.h" static const int PORT = 8713; @@ -3316,6 +3318,167 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } +class RdmaAdmissionTest : public RdmaTest { +protected: + void SetUp() override { + _saved_mode = rdma::GetRdmaRecoveryMode(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + } + + void TearDown() override { + StopServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(_saved_mode); + RdmaTest::TearDown(); + } + + void CallEchoOverFallback() { + Channel channel; + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.connect_timeout_ms = 1000; + options.timeout_ms = 1000; + options.max_retry = 0; + ASSERT_EQ(0, channel.Init(g_ep, &options)); + Controller cntl; + test::EchoRequest req; + test::EchoResponse res; + req.set_message("runtime recovery"); + test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ("MyEchoService", res.message()); + SocketUniquePtr s; + ASSERT_EQ(0, Socket::Address(cntl._single_server_id, &s)); + EXPECT_EQ(RdmaTransport::RDMA_OFF, RdmaTransportOf(s)->_rdma_state); + EXPECT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, + RdmaTransportOf(s)->_rdma_ep->_state); + EXPECT_FALSE(s->Failed()); + } + +private: + rdma::RdmaRecoveryMode _saved_mode; +}; + +TEST_F(RdmaAdmissionTest, tcp_only_carries_rpc_to_tcp_server) { + StartServer(false); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, cooled_peer_carries_rpc_over_tcp) { + StartServer(); + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + policy.ReportPeerFailure(peer, epoch); + ASSERT_FALSE(policy.AllowNewConnection(peer)); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +static int PollRnrCompletion(ibv_cq* cq, int, ibv_wc* wc) { + bool* delivered = static_cast(cq->cq_context); + if (*delivered) { + return 0; + } + *delivered = true; + memset(wc, 0, sizeof(*wc)); + wc->status = IBV_WC_RNR_RETRY_EXC_ERR; + return 1; +} + +TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { + StartServer(); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + SocketOptions cq_options; + cq_options.user = ep; + SocketId cq_sid; + ASSERT_EQ(0, Socket::Create(cq_options, &cq_sid)); + SocketUniquePtr cq_socket; + ASSERT_EQ(0, Socket::Address(cq_sid, &cq_socket)); + + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + bool delivered = false; + ibv_context context{}; + context.ops.poll_cq = PollRnrCompletion; + ibv_cq cq{}; + cq.context = &context; + cq.cq_context = &delivered; + rdma::RdmaResource resource; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->_cq_sid = cq_sid; + ep->_recovery_epoch = epoch; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const bool saved_polling = rdma::FLAGS_rdma_use_polling; + rdma::FLAGS_rdma_use_polling = true; + + rdma::RdmaEndpoint::PollCq(cq_socket.get()); + EXPECT_TRUE(delivered); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); + + // The test owns the mock CQ and resource; ordinary recycling owns neither. + ep->_resource = nullptr; + ep->_cq_sid = INVALID_SOCKET_ID; + resource.polling_cq = nullptr; + cq_socket->_user = nullptr; + cq_socket->SetFailed(); + rdma::FLAGS_rdma_use_polling = saved_polling; + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, server_declines_v2_and_v3_when_tcp_only) { + StartServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + for (int version : {2, 3}) { + butil::fd_guard fd; + ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); + if (version == 2) { + uint8_t hello[rdma::HELLO_V2_MSG_LEN_MIN]; + MakeV2ClientHello(hello); + ASSERT_TRUE(WriteAll(fd, hello, sizeof(hello))); + } else { + const std::string hello = MakeV3Packet(MakeValidV3Hello()); + ASSERT_TRUE(WriteAll(fd, hello.data(), hello.size())); + } + Socket* s = WaitForServerSocket(); + ASSERT_TRUE(s != nullptr); + auto* transport = RdmaTransportOf(s); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::S_ACK_WAIT, transport); + EXPECT_EQ(RdmaTransport::RDMA_OFF, transport->_rdma_state); + EXPECT_EQ(nullptr, transport->_rdma_ep->_resource); + const uint32_t ack = 0; + ASSERT_TRUE(WriteAll(fd, &ack, sizeof(ack))); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::FALLBACK_TCP, transport); + EXPECT_FALSE(s->Failed()); + fd.reset(-1); + ASSERT_TRUE(WaitForServerSocketGone()); + } +} + +TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RETRY_EXC_ERR)); + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RNR_RETRY_EXC_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_SUCCESS)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_WR_FLUSH_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_LOC_PROT_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); +} + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { From 15ad2ffb85dc986a77e2feac41c0bee5e63d32dd Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:23:38 -0700 Subject: [PATCH 04/29] Unify RDMA failure reporting and add recovery diagnostics --- docs/cn/rdma_runtime_recovery.md | 6 + example/BUILD.bazel | 10 +- example/rdma_performance/CMakeLists.txt | 4 + example/rdma_performance/Makefile | 11 +- example/rdma_performance/RECOVERY.md | 94 ++++++++++++++ example/rdma_performance/recovery_client.cpp | 122 +++++++++++++++++++ src/brpc/rdma/rdma_endpoint.cpp | 93 ++++++++++++-- src/brpc/rdma/rdma_endpoint.h | 8 ++ src/brpc/rdma/rdma_helper.cpp | 8 +- src/brpc/rdma/rdma_recovery_policy.cpp | 101 ++++++++++++++- src/brpc/rdma/rdma_recovery_policy.h | 29 ++++- test/brpc_rdma_recovery_unittest.cpp | 30 +++++ test/brpc_rdma_unittest.cpp | 107 ++++++++++++++++ 13 files changed, 605 insertions(+), 18 deletions(-) create mode 100644 example/rdma_performance/RECOVERY.md create mode 100644 example/rdma_performance/recovery_client.cpp diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index 058a2088f6..acb9a72b6a 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -29,6 +29,12 @@ under the License. - AUTO 下,客户端收到 RETRY_EXC / RNR_RETRY_EXC 后,按远端地址冷却 5 秒。 - 当前设备的 PORT_ERR 阻止新 RDMA 建链,端口恢复且冷却结束后重新准入。 - 对端冷却缓存最多 1024 项,满时淘汰最早到期项;模式切换用 epoch 过滤旧连接错误。 +- 故障上报覆盖 QP/CQ 异步错误、CQ 轮询/事件/通知错误及链路类发送错误,同一连接只计一次。 +- QP/CQ 绑定所属 socket,异步事件核对当前资源;发生运行期错误的资源直接释放。 + +`GetRdmaRecoveryStatus()` 和 `/vars/rdma_recovery_*` 暴露冷却状态、分原因故障计数、 +TCP 回落准入数及 RDMA 协商数。实机步骤见 +[RDMA 双机故障恢复测试](../../example/rdma_performance/RECOVERY.md)。 通过进程级 API 配置: diff --git a/example/BUILD.bazel b/example/BUILD.bazel index 05eda2080b..9049cf07d7 100644 --- a/example/BUILD.bazel +++ b/example/BUILD.bazel @@ -123,6 +123,14 @@ cc_binary( ], ) +cc_binary( + name = "rdma_recovery_client", + srcs = ["rdma_performance/recovery_client.cpp"], + includes = ["rdma_performance"], + copts = COPTS + ["-DBRPC_WITH_RDMA=1"], + deps = [":cc_rdma_performance_proto", "//:brpc"], +) + cc_binary( name = "ubring_performance_server", srcs = [ @@ -162,4 +170,4 @@ cc_binary( deps = [ "//:brpc", ], -) \ No newline at end of file +) diff --git a/example/rdma_performance/CMakeLists.txt b/example/rdma_performance/CMakeLists.txt index 213cff0f24..7e0e70f724 100644 --- a/example/rdma_performance/CMakeLists.txt +++ b/example/rdma_performance/CMakeLists.txt @@ -40,3 +40,7 @@ brpc_example_configure_target(server) target_link_libraries(client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) target_link_libraries(server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +add_executable(recovery_client recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) +brpc_example_configure_target(recovery_client) +target_link_libraries(recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/rdma_performance/Makefile b/example/rdma_performance/Makefile index c112f1c330..58b40acf8b 100644 --- a/example/rdma_performance/Makefile +++ b/example/rdma_performance/Makefile @@ -59,7 +59,7 @@ endif .PHONY:all ifdef WITH_RDMA -all: client server +all: client server recovery_client else all: endif @@ -67,7 +67,7 @@ endif .PHONY:clean clean: @echo "> Cleaning" - rm -rf client server $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) + rm -rf client server recovery_client recovery_client.o $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) client:$(PROTO_OBJS) $(CLIENT_OBJS) @echo "> Linking $@" @@ -85,6 +85,13 @@ else $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ endif +recovery_client:$(PROTO_OBJS) recovery_client.o +ifneq ("$(LINK_SO)", "") + $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ +else + $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ +endif + %.pb.cc %.pb.h:%.proto @echo "> Generating $@" $(PROTOC) --cpp_out=. --proto_path=. $(PROTOC_EXTRA_ARGS) $< diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md new file mode 100644 index 0000000000..9b074ce8c1 --- /dev/null +++ b/example/rdma_performance/RECOVERY.md @@ -0,0 +1,94 @@ + + +# RDMA 双机故障恢复测试 + +两台 Linux 主机运行 RDMA 服务端和 recovery_client,TCP 地址通过独立可达的网络通信, +故障注入作用于 RDMA 测试链路。记录设备、端口、GID、驱动和固件版本。 + +## 编译 + +在仓库根目录执行: + +```bash +cmake -S . -B build -DWITH_RDMA=ON -DBUILD_SHARED_LIBS=OFF +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/rdma_performance -B build/recovery -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/recovery --target server recovery_client -j8 +``` + +Bazel 入口为 `//example:rdma_recovery_client`,Make 入口为示例目录中的 `recovery_client`。 + +## 运行与切换 + +主机 B 启动服务端,按环境选择 RDMA 设备及 GID: + +```bash +./build/recovery/server --port=8002 --use_rdma=true --rdma_device=mlx5_0 --rdma_gid_index=3 +``` + +主机 A 启动客户端,将 B_TCP_IP 替换为服务端 TCP 地址: + +```bash +printf 'auto\n' > /tmp/brpc-rdma-mode +./build/recovery/recovery_client --server=B_TCP_IP:8002 --seconds=180 \ + --mode_file=/tmp/brpc-rdma-mode --connection_type=short \ + --rdma_device=mlx5_0 --rdma_gid_index=3 +``` + +客户端持续发送 RPC,失败后继续下一次调用。每秒输出累计成功数、失败数、RDMA 协商数、 +TCP 回落准入数、冷却对端数和端口状态。运行中修改模式文件: + +```bash +printf 'tcp_only\n' > /tmp/brpc-rdma-mode +printf 'auto\n' > /tmp/brpc-rdma-mode +``` + +`short` 为每次调用建立连接,便于验证准入和冷却后的 RDMA 重建。 +`single` 用于观察存量连接:已回落的 TCP 连接继续复用,模式切换影响后续新连接。 + +客户端诊断入口: + +```bash +curl 'http://127.0.0.1:8001/vars/rdma_recovery_*' +``` + +`rdma_recovery_status` 提供模式、epoch、端口状态、冷却剩余时间和分原因故障计数; +`rdma_recovery_tcp_fallback_admissions` 为策略选择 TCP 的累计准入次数。 +计数以连接准入为单位,独立于 RPC 成功数。 + +## 测试步骤 + +| 场景 | 操作 | 观测 | +| --- | --- | --- | +| 健康链路 | AUTO 运行 | RPC 成功,RDMA 协商数持续增加 | +| 运维降级 | 文件改为 tcp_only | 后续连接成功,TCP 准入数增加,RDMA 协商数停止增加 | +| 恢复模式 | 文件改回 auto | short 模式下 RDMA 协商数恢复增加 | +| 端口故障 | 关闭客户端 RDMA 测试端口,保持 TCP 地址可达 | port_active=false,新连接经 TCP 成功 | +| 端口恢复 | 恢复测试端口,等待 cooldown_ms | 端口变为 active,冷却结束后 RDMA 协商数恢复增加 | +| 在途故障 | single 模式下断开 RDMA 链路 | 记录在途失败和恢复时延,后续连接经 TCP 继续调用 | +| 对端故障 | 中断服务端 RDMA 数据路径,保持 TCP 服务 | retry_exhausted / qp_fatal 等计数增加,对端进入冷却 | + +RNR 测试需要接收端故障注入,暂停补充接收 WR 并突破接收 credit 保护后观察 rnr 计数。 +普通拥塞可能只表现为背压;源码 mock CQE 测试可确定性验证 RNR 分支。 + +使用 `--use_rdma=false` 与 TCP 服务端可做无网卡烟测。 +客户端退出码 0 表示至少一次 RPC 成功,2 表示整个测试均失败;故障窗口内的失败数单独查看。 +验收记录故障前后成功率、恢复时延和各原因计数;性能基线使用原 performance client。 diff --git a/example/rdma_performance/recovery_client.cpp b/example/rdma_performance/recovery_client.cpp new file mode 100644 index 0000000000..9b4a36badf --- /dev/null +++ b/example/rdma_performance/recovery_client.cpp @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/rdma/rdma_recovery_policy.h" +#include "butil/time.h" +#include "test.pb.h" + +DEFINE_string(server, "127.0.0.1:8002", "Performance test server address"); +DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); +DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); +DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); +DEFINE_bool(use_rdma, true, "Use RDMA; false runs a TCP smoke test"); +DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); +DEFINE_int32(interval_ms, 100, "Pause between RPCs"); +DEFINE_int32(timeout_ms, 1000, "RPC deadline"); +DEFINE_int32(cooldown_ms, 5000, "Path cooldown"); +DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); + +static bool SetMode(const std::string& value) { + using brpc::rdma::RdmaRecoveryMode; + if (value == "legacy") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::LEGACY); + } + if (value == "auto") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::AUTO); + } + if (value == "tcp_only") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::TCP_ONLY); + } + return false; +} + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || + !brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms) || + !SetMode(FLAGS_mode)) { + std::cerr << "Invalid duration, interval, timeout, cooldown or mode\n"; + return 1; + } + brpc::Channel channel; + brpc::ChannelOptions options; + options.socket_mode = FLAGS_use_rdma ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = FLAGS_timeout_ms; + options.max_retry = 0; + if (channel.Init(FLAGS_server.c_str(), &options) != 0) { + return 1; + } + if (FLAGS_dummy_port && brpc::StartDummyServerAt(FLAGS_dummy_port) != 0) { + return 1; + } + std::string mode = FLAGS_mode; + uint64_t succeeded = 0; + uint64_t failed = 0; + const int64_t start = butil::monotonic_time_us(); + int64_t next_report = start; + std::cout << "seconds,mode,success,failure,rdma_negotiated,tcp_fallback_admissions," + "cooled_peers,port_active,port_cooldown_ms\n"; + while (!brpc::IsAskedToQuit() && + butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { + if (!FLAGS_mode_file.empty()) { + std::ifstream input(FLAGS_mode_file); + std::string value; + if (input >> value && value != mode) { + if (SetMode(value)) { + mode = value; + } else { + std::cerr << "Ignoring invalid recovery mode: " << value << '\n'; + } + } + } + test::PerfTestRequest request; + request.set_echo_attachment(false); + test::PerfTestResponse response; + brpc::Controller cntl; + test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, nullptr); + if (cntl.Failed()) { + ++failed; + std::cerr << "RPC failure: " << cntl.ErrorText() << '\n'; + } else { + ++succeeded; + } + const int64_t now = butil::monotonic_time_us(); + if (now >= next_report) { + const auto s = brpc::rdma::GetRdmaRecoveryStatus(); + std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded + << ',' << failed << ',' << s.negotiated_connections << ',' + << s.tcp_only_admissions + s.port_cooldown_admissions + + s.peer_cooldown_admissions + << ',' << s.cooled_peers << ',' << s.port_active << ',' + << s.port_cooldown_ms << std::endl; + next_report = now + 1000000; + } + usleep(static_cast(FLAGS_interval_ms) * 1000); + } + std::cout << "completed success=" << succeeded << " failure=" << failed << std::endl; + return succeeded ? 0 : 2; +} diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 33eebaa08c..3d78e47a75 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -133,6 +133,7 @@ RdmaEndpoint::RdmaEndpoint(Socket* s) , _state(UNINIT) , _handshake_version(0) , _recovery_epoch(0) + , _runtime_failure(RdmaFailureReason::NONE) , _resource(nullptr) , _send_cq_events(0) , _recv_cq_events(0) @@ -181,6 +182,7 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; _recovery_epoch = 0; + _runtime_failure.store(RdmaFailureReason::NONE, butil::memory_order_relaxed); _outgoing_ece.reset(); _resource = nullptr; _send_cq_events = 0; @@ -242,6 +244,50 @@ void RdmaConnect::StartConnect(const Socket* socket, void RdmaConnect::StopConnect(Socket* socket) { } +void RdmaEndpoint::ReportFailure(RdmaFailureReason reason) { + RdmaFailureReason expected = RdmaFailureReason::NONE; + if (reason == RdmaFailureReason::NONE || + !_runtime_failure.compare_exchange_strong(expected, reason)) { + return; + } + if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { + return; + } + GlobalRdmaRecoveryPolicy().ReportPeerFailure( + _socket->CreatedByConnect() ? + butil::endpoint2str(_socket->remote_side()).c_str() : "", + _recovery_epoch, reason); + LOG(WARNING) << "RDMA runtime failure=" << RdmaFailureReasonName(reason) + << " socket=" << _socket->id() + << " port=" << static_cast(GetRdmaPortNum()) + << " epoch=" << _recovery_epoch; +} + +void RdmaEndpoint::OnAsyncError(Socket* socket, const ibv_async_event& event) { + auto* transport = dynamic_cast(socket->_transport.get()); + RdmaEndpoint* ep = transport ? transport->GetRdmaEp() : nullptr; + if (!ep || !ep->_resource) { + return; + } + RdmaFailureReason reason; + if (event.event_type == IBV_EVENT_CQ_ERR) { + if (event.element.cq != ep->_resource->send_cq && + event.element.cq != ep->_resource->recv_cq && + event.element.cq != ep->_resource->polling_cq) { + return; + } + reason = RdmaFailureReason::CQ_ERROR; + } else { + if (event.element.qp != ep->_resource->qp) { + return; + } + reason = event.event_type == IBV_EVENT_QP_FATAL ? + RdmaFailureReason::QP_FATAL : RdmaFailureReason::ACCESS_ERROR; + } + ep->ReportFailure(reason); + socket->SetFailed(ERDMA, "RDMA async error: %s", RdmaFailureReasonName(reason)); +} + void RdmaConnect::Run() { _done(errno, _data); } @@ -587,6 +633,7 @@ void* RdmaEndpoint::ProcessHandshakeAtClient(void* arg) { if (rdma_transport->_rdma_state == RdmaTransport::RDMA_ON) { ep->_state.store(ESTABLISHED, butil::memory_order_release); + GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); // The handshake is over, so the QP stream may start parsing now. if (ep->StartCqEvents() < 0) { const int saved_errno = errno; @@ -773,6 +820,7 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s << ") on " << s->description(); rdma_transport->_rdma_state = RdmaTransport::RDMA_ON; ep->_state.store(ESTABLISHED, butil::memory_order_release); + GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); s->reset_parsing_context(nullptr); // Two things are deliberately not done here. @@ -985,6 +1033,9 @@ ssize_t RdmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { std::ostringstream oss; DebugInfo(oss, ", "); LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); + if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { + ReportFailure(RdmaFailureReason::SEND_POST); + } errno = err; return -1; } @@ -1035,6 +1086,10 @@ int RdmaEndpoint::SendImm(uint32_t imm) { // We use other way to guarantee the Send Queue is not full. // So we just consider this error as an unrecoverable error. LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); + if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { + ReportFailure(RdmaFailureReason::SEND_POST); + } + errno = err; return -1; } @@ -1052,8 +1107,7 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (0 == wc.wr_id) { _sq_imm_window_size += 1; // If there are any unacknowledged recvs, send an ack. - SendAck(0); - return 0; + return SendAck(0); } // Update SQ window. uint16_t wnd_to_update = wc.wr_id; @@ -1106,8 +1160,8 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (wc.byte_len > 0) { - SendAck(1); + if (wc.byte_len > 0 && SendAck(1) < 0) { + return -1; } return wc.byte_len; } @@ -1342,6 +1396,8 @@ int RdmaEndpoint::DoAllocateResources() { return -1; } + BindResourceToSocket(); + if (!FLAGS_rdma_use_polling) { if (0 != ReqNotifyCq(true, false)) { return -1; @@ -1367,6 +1423,20 @@ int RdmaEndpoint::DoAllocateResources() { return 0; } +void RdmaEndpoint::BindResourceToSocket() { + void* owner = reinterpret_cast(_socket->id()); + _resource->qp->qp_context = owner; + if (_resource->send_cq) { + _resource->send_cq->cq_context = owner; + } + if (_resource->recv_cq) { + _resource->recv_cq->cq_context = owner; + } + if (_resource->polling_cq) { + _resource->polling_cq->cq_context = owner; + } +} + int RdmaEndpoint::StartCqEvents() { if (InputMessengerProcessor::STREAM_NONE != _socket->parsing_stream_type()) { LOG(WARNING) << "StartCqEvents() called while " << *_socket << " is parsing"; @@ -1552,7 +1622,8 @@ void RdmaEndpoint::DeallocateResources() { PollerRemoveCqSid(); } bool move_to_rdma_resource_list = false; - if (_sq_size <= FLAGS_rdma_prepared_qp_size && + if (_runtime_failure.load(butil::memory_order_relaxed) == RdmaFailureReason::NONE && + _sq_size <= FLAGS_rdma_prepared_qp_size && _rq_size <= FLAGS_rdma_prepared_qp_size && FLAGS_rdma_prepared_qp_cnt > 0) { ibv_qp_attr attr; @@ -1659,6 +1730,7 @@ int RdmaEndpoint::GetAndAckEvents(SocketUniquePtr& s) { if (errno != EAGAIN) { const int saved_errno = errno; PLOG(ERROR) << "Fail to get cq event from " << s->description(); + ReportFailure(RdmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to get cq event from %s: %s", s->description().c_str(), berror(saved_errno)); return -1; @@ -1699,6 +1771,7 @@ int RdmaEndpoint::ReqNotifyCq(bool send_cq, bool fatal_on_error) { PLOG(WARNING) << "Fail to arm " << (send_cq ? "send" : "recv") << " CQ comp channel from " << _socket->description(); if (fatal_on_error) { + ReportFailure(RdmaFailureReason::CQ_NOTIFY); _socket->SetFailed(err, "Fail to arm %s CQ channel from %s: %s", send_cq ? "send" : "recv", _socket->description().c_str(), berror(err)); @@ -1752,6 +1825,7 @@ void RdmaEndpoint::PollCq(Socket* m) { if (cnt < 0) { const int saved_errno = errno; PLOG(WARNING) << "Fail to poll cq: " << s->description(); + ep->ReportFailure(RdmaFailureReason::CQ_POLL); s->SetFailed(saved_errno, "Fail to poll cq from %s: %s", s->description().c_str(), berror(saved_errno)); return; @@ -1820,10 +1894,11 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { - if (s->CreatedByConnect() && IsRdmaPathFailure(wc[i].status)) { - GlobalRdmaRecoveryPolicy().ReportPeerFailure( - butil::endpoint2str(s->remote_side()).c_str(), - ep->_recovery_epoch); + if (IsRdmaPathFailure(wc[i].status)) { + ep->ReportFailure(wc[i].status == IBV_WC_RNR_RETRY_EXC_ERR ? + RdmaFailureReason::RNR : RdmaFailureReason::RETRY_EXHAUSTED); + } else if (wc[i].status != IBV_WC_WR_FLUSH_ERR) { + ep->ReportFailure(RdmaFailureReason::ACCESS_ERROR); } PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index eff186be50..85d3b280c9 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -32,6 +32,7 @@ #include "butil/containers/optional.h" #include "brpc/socket.h" #include "brpc/rdma/rdma_handshake_server.h" +#include "brpc/rdma/rdma_recovery_policy.h" namespace brpc { @@ -136,6 +137,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // Real handshake for RDMA-mode sockets. static ParseResult ExecuteServerHandshake(butil::IOBuf* source, Socket* socket); + // Called with an addressed Socket held until its async event is acknowledged. + static void OnAsyncError(Socket* socket, const ibv_async_event& event); + // Initialize polling mode static int PollingModeInitialize(bthread_tag_t tag, std::function callback, @@ -145,6 +149,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); static void PollingModeRelease(bthread_tag_t tag); private: + void ReportFailure(RdmaFailureReason reason); + void BindResourceToSocket(); + enum State { UNINIT = 0x0, C_ALLOC_QPCQ = 0x1, @@ -309,6 +316,7 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // Admission epoch used to discard failures from an earlier recovery mode. uint64_t _recovery_epoch; + butil::atomic _runtime_failure; // ECE payload to advertise in the next local hello: // Client: the locally queried ECE capabilities (filled diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index f0ff394a37..1ff7884acd 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -269,7 +269,7 @@ static void OnRdmaAsyncEvent(Socket* m) { SocketId sid = (SocketId)event.element.qp->qp_context; SocketUniquePtr s; if (Socket::Address(sid, &s) == 0) { - s->SetFailed(ERDMA, "QP fatal error"); + RdmaEndpoint::OnAsyncError(s.get(), event); LOG(WARNING) << "Receive a QP fatal error on " << s->description(); } // NOTE: @@ -281,7 +281,11 @@ static void OnRdmaAsyncEvent(Socket* m) { break; } case IBV_EVENT_CQ_ERR: { - LOG(WARNING) << "CQ overruns, the connection will be stopped."; + SocketUniquePtr s; + const SocketId sid = reinterpret_cast(event.element.cq->cq_context); + if (Socket::Address(sid, &s) == 0) { + RdmaEndpoint::OnAsyncError(s.get(), event); + } IbvAckAsyncEvent(&event); break; } diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp index 801605cb90..fbaa4d6fd2 100644 --- a/src/brpc/rdma/rdma_recovery_policy.cpp +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -17,10 +17,27 @@ #include "brpc/rdma/rdma_recovery_policy.h" #include "butil/time.h" +#include "bvar/passive_status.h" namespace brpc { namespace rdma { +const char* RdmaFailureReasonName(RdmaFailureReason reason) { + switch (reason) { + case RdmaFailureReason::NONE: return "none"; + case RdmaFailureReason::RETRY_EXHAUSTED: return "retry_exhausted"; + case RdmaFailureReason::RNR: return "rnr"; + case RdmaFailureReason::QP_FATAL: return "qp_fatal"; + case RdmaFailureReason::CQ_ERROR: return "cq_error"; + case RdmaFailureReason::CQ_POLL: return "cq_poll"; + case RdmaFailureReason::CQ_EVENT: return "cq_event"; + case RdmaFailureReason::CQ_NOTIFY: return "cq_notify"; + case RdmaFailureReason::SEND_POST: return "send_post"; + case RdmaFailureReason::ACCESS_ERROR: return "access_error"; + default: return "unknown"; + } +} + RdmaRecoveryPolicy::RdmaRecoveryPolicy(Clock clock, size_t max_peers) : _clock(clock) , _max_peers(max_peers) @@ -69,10 +86,12 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, return true; } if (_mode == RdmaRecoveryMode::TCP_ONLY) { + ++_status.tcp_only_admissions; return false; } const int64_t now = _clock(); if (!_port_active || now < _port_retry_at) { + ++_status.port_cooldown_admissions; return false; } auto it = _peers.find(peer); @@ -80,6 +99,7 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, return true; } if (now < it->second) { + ++_status.peer_cooldown_admissions; return false; } _peers.erase(it); @@ -87,10 +107,16 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, } void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, - uint64_t epoch) { + uint64_t epoch, + RdmaFailureReason reason) { std::lock_guard lock(_mutex); - if (_mode != RdmaRecoveryMode::AUTO || epoch != _epoch || - peer.empty() || _max_peers == 0) { + if (epoch != _epoch || reason <= RdmaFailureReason::NONE || + reason >= RdmaFailureReason::COUNT) { + return; + } + ++_status.failures[static_cast(reason)]; + if (_mode != RdmaRecoveryMode::AUTO || peer.empty() || _max_peers == 0 || + reason == RdmaFailureReason::ACCESS_ERROR) { return; } const int64_t now = _clock(); @@ -109,6 +135,7 @@ void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, } } _peers.erase(oldest); + ++_status.cache_evictions; } _peers[peer] = now + _cooldown_us; } @@ -117,14 +144,78 @@ void RdmaRecoveryPolicy::ReportPortState(bool active) { std::lock_guard lock(_mutex); _port_active = active; if (!active) { + ++_status.port_errors; _port_retry_at = _clock() + _cooldown_us; } } +RdmaRecoveryStatus RdmaRecoveryPolicy::GetStatus() { + std::lock_guard lock(_mutex); + RdmaRecoveryStatus result = _status; + result.mode = _mode; + result.epoch = _epoch; + result.port_active = _port_active; + const int64_t now = _clock(); + result.port_cooldown_ms = _port_retry_at > now ? + (_port_retry_at - now + 999) / 1000 : 0; + for (const auto& peer : _peers) { + result.cooled_peers += peer.second > now; + } + return result; +} + +void RdmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { + std::lock_guard lock(_mutex); + if (epoch == _epoch) { + ++_status.negotiated_connections; + } +} + +static void DescribeRecovery(std::ostream& os, void* arg) { + const auto s = static_cast(arg)->GetStatus(); + os << "{\"mode\":\"" + << (s.mode == RdmaRecoveryMode::AUTO ? "auto" : + s.mode == RdmaRecoveryMode::TCP_ONLY ? "tcp_only" : "legacy") + << "\",\"epoch\":" << s.epoch + << ",\"port_active\":" << (s.port_active ? "true" : "false") + << ",\"port_cooldown_ms\":" << s.port_cooldown_ms + << ",\"cooled_peers\":" << s.cooled_peers + << ",\"tcp_only_admissions\":" << s.tcp_only_admissions + << ",\"port_cooldown_admissions\":" << s.port_cooldown_admissions + << ",\"peer_cooldown_admissions\":" << s.peer_cooldown_admissions + << ",\"port_errors\":" << s.port_errors + << ",\"cache_evictions\":" << s.cache_evictions + << ",\"negotiated_connections\":" << s.negotiated_connections + << ",\"failures\":{"; + for (size_t i = 1; i < static_cast(RdmaFailureReason::COUNT); ++i) { + if (i > 1) { + os << ','; + } + os << '"' << RdmaFailureReasonName(static_cast(i)) + << "\":" << s.failures[i]; + } + os << "}}"; +} + +struct RecoveryMetrics { + explicit RecoveryMetrics(RdmaRecoveryPolicy* policy) + : status("rdma_recovery_status", DescribeRecovery, policy) + , fallbacks("rdma_recovery_tcp_fallback_admissions", [](void* arg) { + const auto s = static_cast(arg)->GetStatus(); + return s.tcp_only_admissions + s.port_cooldown_admissions + + s.peer_cooldown_admissions; + }, policy) { + } + bvar::PassiveStatus status; + bvar::PassiveStatus fallbacks; +}; + RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { // Async event callbacks may outlive ordinary static destructors. static RdmaRecoveryPolicy* policy = new RdmaRecoveryPolicy(butil::monotonic_time_us); + static RecoveryMetrics* metrics = new RecoveryMetrics(policy); + (void)metrics; return *policy; } @@ -140,5 +231,9 @@ bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms) { return GlobalRdmaRecoveryPolicy().SetCooldownMs(cooldown_ms); } +RdmaRecoveryStatus GetRdmaRecoveryStatus() { + return GlobalRdmaRecoveryPolicy().GetStatus(); +} + } // namespace rdma } // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h index d5f430d6db..651cf8ccb5 100644 --- a/src/brpc/rdma/rdma_recovery_policy.h +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -29,6 +29,28 @@ namespace rdma { enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; +enum class RdmaFailureReason { + NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, + CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, COUNT +}; + +const char* RdmaFailureReasonName(RdmaFailureReason reason); + +struct RdmaRecoveryStatus { + RdmaRecoveryMode mode = RdmaRecoveryMode::LEGACY; + uint64_t epoch = 0; + bool port_active = true; + int64_t port_cooldown_ms = 0; + size_t cooled_peers = 0; + uint64_t failures[static_cast(RdmaFailureReason::COUNT)] = {}; + uint64_t tcp_only_admissions = 0; + uint64_t port_cooldown_admissions = 0; + uint64_t peer_cooldown_admissions = 0; + uint64_t port_errors = 0; + uint64_t cache_evictions = 0; + uint64_t negotiated_connections = 0; +}; + // Admission policy for new connections on the process's selected RDMA port. // An empty peer selects server admission, which only checks the local port. class RdmaRecoveryPolicy { @@ -40,8 +62,11 @@ class RdmaRecoveryPolicy { // Positive milliseconds, at most INT32_MAX. Applies to future failures. bool SetCooldownMs(int64_t cooldown_ms); bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); - void ReportPeerFailure(const std::string& peer, uint64_t epoch); + void ReportPeerFailure(const std::string& peer, uint64_t epoch, + RdmaFailureReason reason = RdmaFailureReason::RETRY_EXHAUSTED); void ReportPortState(bool active); + RdmaRecoveryStatus GetStatus(); + void RecordNegotiatedConnection(uint64_t epoch); private: std::mutex _mutex; @@ -53,6 +78,7 @@ class RdmaRecoveryPolicy { bool _port_active; int64_t _port_retry_at; std::map _peers; + RdmaRecoveryStatus _status; }; // Process-lifetime policy, shared by client and server handshake admission. @@ -60,6 +86,7 @@ RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy(); bool SetRdmaRecoveryMode(RdmaRecoveryMode mode); RdmaRecoveryMode GetRdmaRecoveryMode(); bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms); +RdmaRecoveryStatus GetRdmaRecoveryStatus(); } // namespace rdma } // namespace brpc diff --git a/test/brpc_rdma_recovery_unittest.cpp b/test/brpc_rdma_recovery_unittest.cpp index 8470230e57..734fe794e2 100644 --- a/test/brpc_rdma_recovery_unittest.cpp +++ b/test/brpc_rdma_recovery_unittest.cpp @@ -151,6 +151,36 @@ TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { EXPECT_FALSE(policy.AllowNewConnection("")); } +TEST_F(RdmaRecoveryTest, reports_reasons_and_admission_counters) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.RecordNegotiatedConnection(epoch); + policy.ReportPeerFailure("a", epoch, RdmaFailureReason::QP_FATAL); + EXPECT_FALSE(policy.AllowNewConnection("a")); + auto status = policy.GetStatus(); + EXPECT_EQ(1u, status.cooled_peers); + EXPECT_EQ(1u, status.negotiated_connections); + EXPECT_EQ(1u, status.peer_cooldown_admissions); + EXPECT_EQ(1u, status.failures[static_cast(RdmaFailureReason::QP_FATAL)]); + policy.ReportPeerFailure("b", epoch, RdmaFailureReason::ACCESS_ERROR); + EXPECT_TRUE(policy.AllowNewConnection("b")); + policy.ReportPortState(false); + EXPECT_FALSE(policy.AllowNewConnection("b")); + status = policy.GetStatus(); + EXPECT_FALSE(status.port_active); + EXPECT_EQ(5000, status.port_cooldown_ms); + EXPECT_EQ(1u, status.port_errors); + EXPECT_EQ(1u, status.port_cooldown_admissions); + now_us += 5000000; + EXPECT_EQ(0u, policy.GetStatus().cooled_peers); + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_EQ(1u, policy.GetStatus().tcp_only_admissions); + policy.ReportPeerFailure("b", epoch, RdmaFailureReason::CQ_ERROR); + EXPECT_EQ(0u, policy.GetStatus().failures[static_cast(RdmaFailureReason::CQ_ERROR)]); +} + } // namespace } // namespace rdma } // namespace brpc diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index bd808ca0bc..f6fe8eeff1 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -3426,6 +3426,11 @@ TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { rdma::FLAGS_rdma_use_polling = true; rdma::RdmaEndpoint::PollCq(cq_socket.get()); + const auto first_report = rdma::GetRdmaRecoveryStatus(); + ep->ReportFailure(rdma::RdmaFailureReason::QP_FATAL); + EXPECT_EQ(first_report.failures[static_cast(rdma::RdmaFailureReason::QP_FATAL)], + rdma::GetRdmaRecoveryStatus().failures[ + static_cast(rdma::RdmaFailureReason::QP_FATAL)]); EXPECT_TRUE(delivered); EXPECT_TRUE(socket->Failed()); EXPECT_FALSE(policy.AllowNewConnection(peer)); @@ -3479,6 +3484,108 @@ TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); } +TEST_F(RdmaAdmissionTest, async_errors_validate_owner_and_report_once) { + for (ibv_event_type type : {IBV_EVENT_QP_FATAL, IBV_EVENT_CQ_ERR, + IBV_EVENT_QP_ACCESS_ERR}) { + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_qp qp{}; + ibv_qp other_qp{}; + ibv_cq cq{}; + ibv_cq other_cq{}; + rdma::RdmaResource resource; + resource.qp = &qp; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->BindResourceToSocket(); + EXPECT_EQ(sid, reinterpret_cast(qp.qp_context)); + EXPECT_EQ(sid, reinterpret_cast(cq.cq_context)); + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const auto before = policy.GetStatus(); + ibv_async_event event{}; + event.event_type = type; + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &other_cq; + } else { + event.element.qp = &other_qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_FALSE(socket->Failed()); + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &cq; + } else { + event.element.qp = &qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_TRUE(socket->Failed()); + const auto reason = type == IBV_EVENT_CQ_ERR ? rdma::RdmaFailureReason::CQ_ERROR : + type == IBV_EVENT_QP_FATAL ? rdma::RdmaFailureReason::QP_FATAL : + rdma::RdmaFailureReason::ACCESS_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + policy.GetStatus().failures[static_cast(reason)]); + EXPECT_EQ(type == IBV_EVENT_QP_ACCESS_ERR, policy.AllowNewConnection(peer)); + ep->ReportFailure(rdma::RdmaFailureReason::RNR); + EXPECT_EQ(before.failures[static_cast(rdma::RdmaFailureReason::RNR)], + policy.GetStatus().failures[static_cast(rdma::RdmaFailureReason::RNR)]); + const std::string exposed = bvar::Variable::describe_exposed("rdma_recovery_status"); + EXPECT_NE(std::string::npos, exposed.find("failures")); + EXPECT_NE(std::string::npos, exposed.find("qp_fatal")); + ep->_resource = nullptr; + resource.qp = nullptr; + resource.polling_cq = nullptr; + } +} + +static int FailAckPost(ibv_qp*, ibv_send_wr*, ibv_send_wr**) { + return EIO; +} + +TEST_F(RdmaAdmissionTest, ack_post_error_propagates_and_cools_peer) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_context context{}; + context.ops.post_send = FailAckPost; + ibv_qp qp{}; + qp.context = &context; + rdma::RdmaResource resource; + resource.qp = &qp; + ep->_resource = &resource; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + ep->_remote_window_capacity = 0; + ep->_new_rq_wrs.store(1); + ibv_wc wc{}; + wc.opcode = IBV_WC_SEND; + EXPECT_EQ(-1, ep->HandleCompletion(wc)); + EXPECT_EQ(EIO, errno); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_EQ(rdma::RdmaFailureReason::SEND_POST, ep->_runtime_failure.load()); + ep->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); +} + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { From 08ed3e2b713851db7db5692e05416467a9a31618 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 03:10:44 -0700 Subject: [PATCH 05/29] Add per-channel RDMA SL and Traffic Class configuration --- docs/cn/rdma.md | 34 +++++++ docs/en/rdma.md | 18 ++++ example/rdma_performance/client.cpp | 12 +++ example/rdma_performance/server.cpp | 4 + src/brpc/acceptor.cpp | 1 + src/brpc/acceptor.h | 2 + src/brpc/channel.cpp | 11 ++- src/brpc/channel.h | 4 + src/brpc/rdma/rdma_endpoint.cpp | 6 +- src/brpc/rdma/rdma_qos.h | 36 +++++++ src/brpc/server.cpp | 6 ++ src/brpc/server.h | 4 + src/brpc/socket.cpp | 7 ++ src/brpc/socket.h | 4 + test/brpc_rdma_unittest.cpp | 143 ++++++++++++++++++++++++++++ 15 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 src/brpc/rdma/rdma_qos.h diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index fd70686bb1..f000b52fb9 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,3 +1,37 @@ +# 业务级 QoS + +通过 `ChannelOptions.rdma_qos` 配置请求方向的 RDMA QoS: + +```cpp +brpc::ChannelOptions options; +options.socket_mode = brpc::SOCKET_MODE_RDMA; +options.rdma_qos.service_level = 3; +options.rdma_qos.traffic_class = (26 << 2) | 2; // DSCP=26, ECN=2 +``` + +`service_level` 范围为 0–15,`traffic_class` 为完整的 8 位 GRH/IP Traffic Class,范围为 0–255, +两者默认均为 0。SL 的调度待遇由网络侧 SL→VL 或优先级映射决定。 +RoCE 部署结合网卡及交换机的 PCP/DSCP 映射配置;示例中的 DSCP/ECN 值按部署策略选择。 + +同一地址下,相同 QoS 的 Channel 可共享连接,不同 SL 或 Traffic Class 使用独立连接及 QP。 +配置随单连接、连接池和短连接传递,并在 QP 重建时重新应用。 + +响应方向通过 `ServerOptions.rdma_qos` 配置: + +```cpp +brpc::ServerOptions options; +options.socket_mode = brpc::SOCKET_MODE_RDMA; +options.rdma_qos.service_level = 3; +options.rdma_qos.traffic_class = 106; +``` + +客户端和服务端分别配置各自的发送方向。设置在 Channel 初始化或 Server 启动时生效, +V2/V3 握手沿用现有格式;TCP 传输按原配置运行。 +连接调试信息提供 `rdma_sl` 和 `rdma_traffic_class`。 + +`example/rdma_performance` 的 client/server 支持 `--rdma_sl`、`--rdma_traffic_class`。 +实机验证时检查 QP 属性和实际报文标记,再比较竞争流量下的吞吐与 P99。 + # 编译 由于RDMA对驱动与硬件有要求,目前仅支持在Linux系统编译并运行RDMA功能。 diff --git a/docs/en/rdma.md b/docs/en/rdma.md index 98ac6981bc..cfac9f83f3 100644 --- a/docs/en/rdma.md +++ b/docs/en/rdma.md @@ -1,3 +1,21 @@ +# Business-level QoS + +Set `ChannelOptions.rdma_qos.service_level` (0–15) and +`ChannelOptions.rdma_qos.traffic_class` (0–255) for outbound requests. +Both default to 0. Traffic Class is the full GRH/IP byte, including DSCP and ECN. +Set `ServerOptions.rdma_qos` independently for outbound responses. + +Different SL or Traffic Class values use separate connections and QPs, while equal +settings can share connections. Settings follow pooled and short sockets and are +applied whenever the QP is brought up. V2/V3 handshake formats stay the same. +TCP transports retain their existing settings. + +The fabric defines how SL maps to VLs and scheduling priority. For RoCE, configure +the NIC and switch PCP/DSCP mappings to match the deployment. The performance +client/server accept `--rdma_sl` and `--rdma_traffic_class`; socket diagnostics show +both applied values. Validate QP attributes and packet markings on hardware before +comparing throughput and P99 under competing workloads. + # Build Since RDMA requires driver and hardware support, only the build on linux is verified. diff --git a/example/rdma_performance/client.cpp b/example/rdma_performance/client.cpp index 8c07eca236..365c61697d 100644 --- a/example/rdma_performance/client.cpp +++ b/example/rdma_performance/client.cpp @@ -42,6 +42,8 @@ DEFINE_string(connection_type, "single", "Connection type of the channel"); DEFINE_string(protocol, "baidu_std", "Protocol type."); DEFINE_string(servers, "0.0.0.0:8002+0.0.0.0:8002", "IP Address of servers"); DEFINE_bool(use_rdma, true, "Use RDMA or not"); +DEFINE_int32(rdma_sl, 0, "Outbound RDMA service level [0,15]"); +DEFINE_int32(rdma_traffic_class, 0, "Outbound RDMA traffic class byte [0,255]"); DEFINE_int32(rpc_timeout_ms, 2000, "RPC call timeout"); DEFINE_int32(test_seconds, 20, "Test running time"); DEFINE_int32(test_iterations, 0, "Test iterations"); @@ -103,6 +105,8 @@ class PerformanceTest { int Init() { brpc::ChannelOptions options; options.socket_mode = FLAGS_use_rdma? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.rdma_qos.service_level = FLAGS_rdma_sl; + options.rdma_qos.traffic_class = FLAGS_rdma_traffic_class; options.protocol = FLAGS_protocol; options.connection_type = FLAGS_connection_type; options.timeout_ms = FLAGS_rpc_timeout_ms; @@ -274,6 +278,14 @@ void Test(int thread_num, int attachment_size) { int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + brpc::RdmaQosOptions qos; + qos.service_level = FLAGS_rdma_sl; + qos.traffic_class = FLAGS_rdma_traffic_class; + if (FLAGS_use_rdma && !qos.IsValid()) { + LOG(ERROR) << "RDMA SL must be in [0,15] and traffic class in [0,255]"; + return 1; + } + // Initialize RDMA environment in advance. if (FLAGS_use_rdma) { brpc::rdma::GlobalRdmaInitializeOrDie(); diff --git a/example/rdma_performance/server.cpp b/example/rdma_performance/server.cpp index 2e93e1eec7..b444b7b02d 100644 --- a/example/rdma_performance/server.cpp +++ b/example/rdma_performance/server.cpp @@ -28,6 +28,8 @@ DEFINE_int32(port, 8002, "TCP Port of this server"); DEFINE_bool(use_rdma, true, "Use RDMA or not"); +DEFINE_int32(rdma_sl, 0, "Outbound RDMA service level [0,15]"); +DEFINE_int32(rdma_traffic_class, 0, "Outbound RDMA traffic class byte [0,255]"); butil::atomic g_last_time(0); @@ -77,6 +79,8 @@ int main(int argc, char* argv[]) { brpc::ServerOptions options; options.socket_mode = FLAGS_use_rdma? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.rdma_qos.service_level = FLAGS_rdma_sl; + options.rdma_qos.traffic_class = FLAGS_rdma_traffic_class; if (server.Start(FLAGS_port, &options) != 0) { LOG(ERROR) << "Fail to start EchoServer"; return -1; diff --git a/src/brpc/acceptor.cpp b/src/brpc/acceptor.cpp index 99280037d5..d19d98ee0a 100644 --- a/src/brpc/acceptor.cpp +++ b/src/brpc/acceptor.cpp @@ -338,6 +338,7 @@ void Acceptor::OnNewConnectionsUntilEAGAIN(Socket* acception) { options.force_ssl = am->_force_ssl; options.initial_ssl_ctx = am->_ssl_ctx; options.socket_mode = am->_socket_mode; + options.rdma_qos = am->_rdma_qos; options.bthread_tag = am->_bthread_tag; if (Socket::Create(options, &socket_id) != 0) { am->ReleaseConnectionSlot(); diff --git a/src/brpc/acceptor.h b/src/brpc/acceptor.h index 01a2b26a9d..2485d9f387 100644 --- a/src/brpc/acceptor.h +++ b/src/brpc/acceptor.h @@ -24,6 +24,7 @@ #include "butil/containers/flat_map.h" #include "brpc/input_messenger.h" #include "brpc/socket_mode.h" +#include "brpc/rdma/rdma_qos.h" namespace brpc { @@ -137,6 +138,7 @@ friend class Server; // Choose to use a certain socket: 0 TCP, 1 RDMA SocketMode _socket_mode; + RdmaQosOptions _rdma_qos; // Acceptor belongs to this tag bthread_tag_t _bthread_tag; diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index de0a1bde49..9a7811f8f3 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -78,7 +78,8 @@ ChannelSSLOptions* ChannelOptions::mutable_ssl_options() { } static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { - if (opt.auth == nullptr && + if (opt.socket_mode == SOCKET_MODE_TCP && + opt.auth == nullptr && !opt.has_ssl_options() && opt.client_host.empty() && opt.device_name.empty() && @@ -139,6 +140,8 @@ static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { } if (opt.socket_mode == SOCKET_MODE_RDMA) { buf.append("|rdma"); + buf.append("|sl=").append(std::to_string(opt.rdma_qos.service_level)); + buf.append("|tc=").append(std::to_string(opt.rdma_qos.traffic_class)); } else if (opt.socket_mode == SOCKET_MODE_URMA) { buf.append("|urma"); } @@ -187,6 +190,10 @@ int Channel::InitChannelOptions(const ChannelOptions* options) { if (options) { // Override default options if user provided one. _options = *options; } + if (_options.socket_mode == SOCKET_MODE_RDMA && !_options.rdma_qos.IsValid()) { + LOG(ERROR) << "RDMA SL must be in [0,15] and traffic class in [0,255]"; + return -1; + } const Protocol* protocol = FindProtocol(_options.protocol); if (nullptr == protocol || !protocol->support_client()) { LOG(ERROR) << "Channel does not support the protocol"; @@ -407,6 +414,7 @@ int Channel::InitSingle(const butil::EndPoint& server_addr_and_port, opt.local_side = client_endpoint; opt.initial_ssl_ctx = ssl_ctx; opt.socket_mode = _options.socket_mode; + opt.rdma_qos = _options.rdma_qos; opt.hc_option = _options.hc_option; opt.device_name = _options.device_name; if (SocketMapInsert(SocketMapKey(server_addr_and_port, sig), @@ -453,6 +461,7 @@ int Channel::Init(const char* ns_url, ns_opt.succeed_without_server = _options.succeed_without_server; ns_opt.log_succeed_without_server = _options.log_succeed_without_server; ns_opt.socket_option.socket_mode = _options.socket_mode; + ns_opt.socket_option.rdma_qos = _options.rdma_qos; ns_opt.channel_signature = ComputeChannelSignature(_options); ns_opt.socket_option.hc_option = _options.hc_option; ns_opt.socket_option.local_side = client_endpoint; diff --git a/src/brpc/channel.h b/src/brpc/channel.h index a778a1bd2a..b4bff6092b 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -38,6 +38,7 @@ #include "brpc/naming_service_filter.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#include "brpc/rdma/rdma_qos.h" namespace brpc { @@ -111,6 +112,9 @@ struct ChannelOptions { // Default: SOCKET_MODE_TCP SocketMode socket_mode; + // Outbound RDMA QoS for this Channel. Used when socket_mode is RDMA. + RdmaQosOptions rdma_qos; + // Turn on authentication for this channel if `auth' is not nullptr. // Note `auth' will not be deleted by channel and must remain valid when // the channel is being used. diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 0dc41c9dd1..fe3b25e66f 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -1453,9 +1453,9 @@ int RdmaEndpoint::BringUpQp(const ParsedHello& remote, bool is_server) { attr.ah_attr.grh.flow_label = 0; attr.ah_attr.grh.sgid_index = GetRdmaGidIndex(); attr.ah_attr.grh.hop_limit = MAX_HOP_LIMIT; - attr.ah_attr.grh.traffic_class = 0; + attr.ah_attr.grh.traffic_class = _socket->rdma_qos().traffic_class; attr.ah_attr.dlid = remote.lid; - attr.ah_attr.sl = 0; + attr.ah_attr.sl = _socket->rdma_qos().service_level; attr.ah_attr.src_path_bits = 0; attr.ah_attr.static_rate = 0; attr.ah_attr.is_global = 1; @@ -1867,6 +1867,8 @@ std::string RdmaEndpoint::GetStateStr() const { void RdmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece connector) const { os << "rdma_state=ON" + << connector << "rdma_sl=" << _socket->rdma_qos().service_level + << connector << "rdma_traffic_class=" << _socket->rdma_qos().traffic_class << connector << "handshake_state=" << GetStateStr() << connector << "handshake_version=" << static_cast(_handshake_version) << connector << "rdma_sq_imm_window_size=" << _sq_imm_window_size diff --git a/src/brpc/rdma/rdma_qos.h b/src/brpc/rdma/rdma_qos.h new file mode 100644 index 0000000000..e75cdafa61 --- /dev/null +++ b/src/brpc/rdma/rdma_qos.h @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_RDMA_QOS_H +#define BRPC_RDMA_QOS_H + +namespace brpc { + +// Outbound QP attributes. Their scheduling behavior is defined by the fabric. +struct RdmaQosOptions { + int service_level = 0; // SL, range [0, 15]. + int traffic_class = 0; // Full GRH/IP traffic class byte, range [0, 255]. + + bool IsValid() const { + return service_level >= 0 && service_level <= 15 && + traffic_class >= 0 && traffic_class <= 255; + } +}; + +} // namespace brpc + +#endif // BRPC_RDMA_QOS_H diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 19d58ac422..96734ae8d0 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -875,6 +875,11 @@ int Server::StartInternal(const butil::EndPoint& endpoint, const ServerOptions default_opt; const ServerOptions& real_opt = opt ? *opt : default_opt; + if (real_opt.socket_mode == SOCKET_MODE_RDMA && !real_opt.rdma_qos.IsValid()) { + LOG(ERROR) << "RDMA SL must be in [0,15] and traffic class in [0,255]"; + return -1; + } + if (!real_opt.h2_settings.IsValid(true/*log_error*/)) { LOG(ERROR) << "Invalid h2_settings"; return -1; @@ -1162,6 +1167,7 @@ int Server::StartInternal(const butil::EndPoint& endpoint, _am->_socket_mode = _options.socket_mode; _am->_bthread_tag = _options.bthread_tag; } + _am->_rdma_qos = _options.rdma_qos; // Set `_status' to RUNNING before accepting connections // to prevent requests being rejected as ELOGOFF _status = RUNNING; diff --git a/src/brpc/server.h b/src/brpc/server.h index 31dc69af2f..769cee4150 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -46,6 +46,7 @@ #include "brpc/baidu_master_service.h" #include "brpc/rpc_pb_message_factory.h" #include "brpc/socket_mode.h" +#include "brpc/rdma/rdma_qos.h" namespace brpc { @@ -242,6 +243,9 @@ struct ServerOptions { // Default: SOCKET_MODE_TCP SocketMode socket_mode; + // Outbound RDMA QoS for responses on this server's accepted connections. + RdmaQosOptions rdma_qos; + // [CAUTION] This option is for implementing specialized baidu-std proxies, // most users don't need it. Don't change this option unless you fully // understand the description below. diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp index 57cfd2296a..f560320b4b 100644 --- a/src/brpc/socket.cpp +++ b/src/brpc/socket.cpp @@ -716,6 +716,10 @@ void SetSocketBufferOptions(int fd) { // version: from version part of _versioned_nref, must be an EVEN number. // slot: designated by ResourcePool. int Socket::Create(const SocketOptions& options, SocketId* id) { + if (options.socket_mode == SOCKET_MODE_RDMA && !options.rdma_qos.IsValid()) { + errno = EINVAL; + return -1; + } return VersionedRefWithId::Create(id, options); } @@ -731,6 +735,7 @@ int Socket::OnCreated(const SocketOptions& options) { }); // start build the transport _socket_mode = options.socket_mode; + _rdma_qos = options.rdma_qos; _transport = TransportFactory::CreateTransport(options.socket_mode); CHECK(nullptr != _transport); _transport->Init(this, options); @@ -2830,6 +2835,7 @@ int Socket::GetPooledSocket(SocketUniquePtr* pooled_socket) { opt.keytable_pool = _keytable_pool; opt.app_connect = _app_connect; opt.socket_mode = _socket_mode; + opt.rdma_qos = _rdma_qos; socket_pool = new SocketPool(opt); SocketPool* expected = nullptr; if (!main_sp->socket_pool.compare_exchange_strong( @@ -2936,6 +2942,7 @@ int Socket::GetShortSocket(SocketUniquePtr* short_socket) { opt.keytable_pool = _keytable_pool; opt.app_connect = _app_connect; opt.socket_mode = _socket_mode; + opt.rdma_qos = _rdma_qos; if (get_client_side_messenger()->Create(opt, &id) != 0 || Address(id, short_socket) != 0) { return -1; diff --git a/src/brpc/socket.h b/src/brpc/socket.h index da701e9ad7..9cd26674c4 100644 --- a/src/brpc/socket.h +++ b/src/brpc/socket.h @@ -43,6 +43,7 @@ #include "brpc/versioned_ref_with_id.h" #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" +#include "brpc/rdma/rdma_qos.h" #include "brpc/input_messenger_processor.h" // InputMessengerProcessor namespace brpc { @@ -309,6 +310,7 @@ struct SocketOptions { bool force_ssl{false}; std::shared_ptr initial_ssl_ctx; SocketMode socket_mode{SOCKET_MODE_TCP}; + RdmaQosOptions rdma_qos; bthread_keytable_pool_t* keytable_pool{nullptr}; SocketConnection* conn{nullptr}; std::shared_ptr app_connect; @@ -632,6 +634,7 @@ friend class TransportFactory; // The socket mode decided at creation. SocketMode socket_mode() const { return _socket_mode; } + const RdmaQosOptions& rdma_qos() const { return _rdma_qos; } // Get an UNUSED socket connecting to the same place as this socket // from the SocketPool of this socket. @@ -999,6 +1002,7 @@ friend class TransportFactory; // Should use SOCKET_MODE_RDMA or SOCKET_MODE_TCP or Other, default is SOCKET_MODE_TCP Transport SocketMode _socket_mode; + RdmaQosOptions _rdma_qos; std::unique_ptr _transport; // Pass from controller, for progressive reading. diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index 6ec54ed90b..1d3ac2c934 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include "butil/endpoint.h" #include "butil/fd_guard.h" @@ -47,6 +48,7 @@ #include "brpc/rdma/rdma_handshake_constants.h" #include "brpc/rdma/rdma_handshake.pb.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/details/load_balancer_with_naming.h" #include "echo.pb.h" static const int PORT = 8713; @@ -3316,6 +3318,147 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } +TEST_F(RdmaTest, qos_rejects_invalid_options_before_connecting) { + for (const auto& values : {std::pair(-1, 0), {16, 0}, {0, -1}, {0, 256}}) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = values.first; + options.rdma_qos.traffic_class = values.second; + Channel channel; + EXPECT_EQ(-1, channel.Init(g_ep, &options)); + ServerOptions server_options; + server_options.socket_mode = SOCKET_MODE_RDMA; + server_options.rdma_qos = options.rdma_qos; + Server server; + EXPECT_EQ(-1, server.Start(PORT, &server_options)); + SocketOptions socket_options; + socket_options.socket_mode = SOCKET_MODE_RDMA; + socket_options.rdma_qos = options.rdma_qos; + SocketId sid; + EXPECT_EQ(-1, Socket::Create(socket_options, &sid)); + EXPECT_EQ(EINVAL, errno); + } +} + +TEST_F(RdmaTest, qos_isolates_channels_and_preserves_equal_config_sharing) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + Channel a, b, same, different_tc, default_rdma, tcp, tcp_with_qos; + ASSERT_EQ(0, default_rdma.Init(g_ep, &options)); + ASSERT_EQ(0, tcp.Init(g_ep, nullptr)); + EXPECT_NE(default_rdma._server_id, tcp._server_id); + options.rdma_qos.service_level = 3; + options.rdma_qos.traffic_class = 106; + ASSERT_EQ(0, a.Init(g_ep, &options)); + ASSERT_EQ(0, same.Init(g_ep, &options)); + EXPECT_EQ(a._server_id, same._server_id); + EXPECT_NE(a._server_id, default_rdma._server_id); + options.rdma_qos.service_level = 4; + ASSERT_EQ(0, b.Init(g_ep, &options)); + EXPECT_NE(a._server_id, b._server_id); + options.rdma_qos.service_level = 3; + options.rdma_qos.traffic_class = 104; + ASSERT_EQ(0, different_tc.Init(g_ep, &options)); + EXPECT_NE(a._server_id, different_tc._server_id); + options.socket_mode = SOCKET_MODE_TCP; + ASSERT_EQ(0, tcp_with_qos.Init(g_ep, &options)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(a._server_id, &socket)); + EXPECT_EQ(3, socket->rdma_qos().service_level); + EXPECT_EQ(106, socket->rdma_qos().traffic_class); +} + +TEST_F(RdmaTest, qos_follows_pooled_short_and_named_connections) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = 7; + options.rdma_qos.traffic_class = 255; + Channel direct, named; + ASSERT_EQ(0, direct.Init(g_ep, &options)); + ASSERT_EQ(0, named.Init(_naming_url.c_str(), "rr", &options)); + SocketUniquePtr main_socket; + ASSERT_EQ(0, Socket::Address(direct._server_id, &main_socket)); + SocketUniquePtr pooled, short_socket, selected; + ASSERT_EQ(0, main_socket->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main_socket->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (Socket* s : {pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(7, s->rdma_qos().service_level); + EXPECT_EQ(255, s->rdma_qos().traffic_class); + } + EXPECT_EQ(main_socket->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); +} + +TEST_F(RdmaTest, qos_server_options_follow_accept_and_restart) { + for (int sl : {3, 7}) { + ServerOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = sl; + options.rdma_qos.traffic_class = 106; + ASSERT_EQ(0, _server.Start(PORT, &options)); + butil::fd_guard fd; + ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); + Socket* socket = WaitForServerSocket(); + ASSERT_TRUE(socket != nullptr); + EXPECT_EQ(sl, socket->rdma_qos().service_level); + EXPECT_EQ(106, socket->rdma_qos().traffic_class); + fd.reset(-1); + ASSERT_TRUE(WaitForServerSocketGone()); + StopServer(); + } +} + +static ibv_ah_attr g_qos_rtr_address; +static int g_qos_rtr_calls; +static int CaptureQosModify(ibv_qp*, ibv_qp_attr* attr, ibv_qp_attr_mask mask) { + if (mask & IBV_QP_AV) { + g_qos_rtr_address = attr->ah_attr; + ++g_qos_rtr_calls; + } + return 0; +} + +TEST_F(RdmaTest, qos_applies_to_rtr_on_each_qp_bringup) { + for (const auto& values : {std::pair(15, 255), {0, 0}}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = values.first; + options.rdma_qos.traffic_class = values.second; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + ibv_qp qp{}; + rdma::RdmaResource resource; + resource.qp = &qp; + ep->_resource = &resource; + ep->_rq_size = 0; + rdma::ParsedHello remote{}; + const bool skip = rdma::g_skip_rdma_init; + const auto modify = rdma::IbvModifyQp; + rdma::g_skip_rdma_init = false; + rdma::IbvModifyQp = CaptureQosModify; + g_qos_rtr_calls = 0; + // Reapply attributes on both client and server bringup. + EXPECT_EQ(0, ep->BringUpQp(remote, false)); + EXPECT_EQ(0, ep->BringUpQp(remote, true)); + rdma::g_skip_rdma_init = skip; + rdma::IbvModifyQp = modify; + EXPECT_EQ(2, g_qos_rtr_calls); + EXPECT_EQ(values.first, g_qos_rtr_address.sl); + EXPECT_EQ(values.second, g_qos_rtr_address.grh.traffic_class); + ep->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); + } +} + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { From ba5e1e178121d999356a0058808bba0da59f9f68 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:53:04 -0700 Subject: [PATCH 06/29] Adapt runtime recovery and TCP fallback to URMA --- docs/cn/rdma_runtime_recovery.md | 6 + docs/cn/urma.md | 25 ++ docs/en/urma.md | 29 ++ example/rdma_performance/RECOVERY.md | 8 + example/rdma_performance/recovery_client.cpp | 41 ++- src/brpc/rdma/rdma_recovery_policy.cpp | 42 ++- src/brpc/rdma/rdma_recovery_policy.h | 2 +- src/brpc/urma/urma_endpoint.cpp | 69 +++- src/brpc/urma/urma_endpoint.h | 7 + src/brpc/urma/urma_handshake.cpp | 7 +- src/brpc/urma/urma_recovery_policy.h | 41 +++ test/brpc_urma_unittest.cpp | 336 +++++++++++++++++++ 12 files changed, 590 insertions(+), 23 deletions(-) create mode 100644 src/brpc/urma/urma_recovery_policy.h diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index acb9a72b6a..d2141c89a5 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -51,6 +51,12 @@ brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::TCP_ONLY); 冷却到期后重新开放准入,TCP 连接沿用现有复用与重建机制。 后续迭代增加存量连接退休、单连接探测、指数退避、发送 watchdog 和多端口主备。 +## URMA 适配 + +URMA 复用准入算法,独立维护模式、epoch、对端缓存和 `urma_recovery_*` 指标。 +接入完成错误、JFC 轮询/事件/通知错误及 WR 提交失败,支持 v2/v3 服务端协商降级。 +配置与行为见 [URMA 运行期故障恢复](urma.md#运行期故障恢复)。 + 以下为完整方案。 ## 当前问题 diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 62e10389a8..c207e7d7a1 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -193,3 +193,28 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing `ContextInitOrDie` 阶段拒绝。 - 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 - 当前实现面向 Linux。 + +## 运行期故障恢复 + +通过进程级 API 控制新连接的传输选择: + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +默认 LEGACY 沿用现有行为;AUTO 在重试耗尽、RNR、JFC 错误或 WR 提交失败后, +将客户端对端地址冷却 5 秒,期间新连接使用 TCP。TCP_ONLY 让新连接直接选择 TCP。 +服务端通过 v2/v3 hello 拒绝 URMA 协商,随后处理 TCP 请求。 + +URMA 和 RDMA 分别维护模式、epoch、冷却缓存和指标。同一连接只上报首个运行期故障; +模式切换过滤旧 epoch 的错误。访问错误单独计数,WR flush 不重复归因, +SQ/接收 credit 耗尽保留 EAGAIN 背压,ACK 提交失败向连接错误处理传播。 + +`GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态和协商数, +`/vars/urma_recovery_tcp_fallback_admissions` 提供 TCP 准入次数。 +模式切换作用于新连接;存量连接使用已协商的传输,冷却到期后重新准入 URMA。 + +`example/rdma_performance/recovery_client` 支持 `--transport=urma`,沿用 `--mode`、`--mode_file` 和 CSV 统计,可用于双机故障注入。 diff --git a/docs/en/urma.md b/docs/en/urma.md index a5b80f45b9..29939960b9 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -207,3 +207,32 @@ is tracked as a future enhancement. rejected at `ContextInitOrDie`. - A hardware data path requires a supported UMDK provider and `liburma`. - The current implementation targets Linux. + +## Runtime recovery + +Configure admission for new connections with the process-level API: + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +LEGACY is the default. AUTO cools a client peer for five seconds after retry +exhaustion, RNR, JFC errors or WR submission failures. New connections use TCP +during cooldown. TCP_ONLY selects TCP for new connections. Servers decline +URMA through the v2/v3 hello and continue processing TCP requests. + +URMA and RDMA keep separate modes, epochs, cooldown caches and metrics. +Each connection reports its first runtime failure; mode changes filter reports +from older epochs. Access errors are counted separately, flush completions do +not add a root cause, and exhausted SQ/receive credits retain EAGAIN backpressure. +ACK submission failures propagate to connection error handling. + +GetUrmaRecoveryStatus() and /vars/urma_recovery_status expose failure reasons, +cooldown state and negotiation counts. /vars/urma_recovery_tcp_fallback_admissions +counts TCP admissions. Mode changes apply to new connections. Existing connections +keep their negotiated transport; URMA admission resumes after cooldown. + +`example/rdma_performance/recovery_client --transport=urma` supports `--mode`, `--mode_file` and CSV statistics for two-host fault injection. diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md index 9b074ce8c1..ed6c22f73d 100644 --- a/example/rdma_performance/RECOVERY.md +++ b/example/rdma_performance/RECOVERY.md @@ -92,3 +92,11 @@ RNR 测试需要接收端故障注入,暂停补充接收 WR 并突破接收 cr 使用 `--use_rdma=false` 与 TCP 服务端可做无网卡烟测。 客户端退出码 0 表示至少一次 RPC 成功,2 表示整个测试均失败;故障窗口内的失败数单独查看。 验收记录故障前后成功率、恢复时延和各原因计数;性能基线使用原 performance client。 + +## URMA + +链接启用 URMA 的 brpc,服务端配置 `ServerOptions.socket_mode = SOCKET_MODE_URMA`, +客户端增加 `--transport=urma`。`--mode`、`--mode_file`、`--cooldown_ms` 和连接类型沿用上述配置。 +CSV 的 `transport_negotiated` 记录选定传输的协商数,诊断入口使用 `/vars/urma_recovery_*`。 +断开 URMA 数据路径并保持 TCP 可达,观察重试耗尽或 RNR 计数、对端冷却和后续 TCP 调用; +冷却到期后使用 short 连接验证 URMA 重新准入。 diff --git a/example/rdma_performance/recovery_client.cpp b/example/rdma_performance/recovery_client.cpp index 9b4a36badf..171a64f500 100644 --- a/example/rdma_performance/recovery_client.cpp +++ b/example/rdma_performance/recovery_client.cpp @@ -24,6 +24,7 @@ #include "brpc/controller.h" #include "brpc/server.h" #include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "test.pb.h" @@ -32,6 +33,7 @@ DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); DEFINE_bool(use_rdma, true, "Use RDMA; false runs a TCP smoke test"); +DEFINE_string(transport, "", "rdma, urma or tcp; empty uses --use_rdma"); DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); DEFINE_int32(interval_ms, 100, "Pause between RPCs"); DEFINE_int32(timeout_ms, 1000, "RPC deadline"); @@ -40,29 +42,43 @@ DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); static bool SetMode(const std::string& value) { using brpc::rdma::RdmaRecoveryMode; + RdmaRecoveryMode mode; if (value == "legacy") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::LEGACY); + mode = RdmaRecoveryMode::LEGACY; + } else if (value == "auto") { + mode = RdmaRecoveryMode::AUTO; + } else if (value == "tcp_only") { + mode = RdmaRecoveryMode::TCP_ONLY; + } else { + return false; } - if (value == "auto") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::AUTO); - } - if (value == "tcp_only") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::TCP_ONLY); - } - return false; + return FLAGS_transport == "urma" ? brpc::urma::SetUrmaRecoveryMode(mode) : + brpc::rdma::SetRdmaRecoveryMode(mode); } int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_transport.empty()) { + FLAGS_transport = FLAGS_use_rdma ? "rdma" : "tcp"; + } + if (FLAGS_transport != "rdma" && FLAGS_transport != "urma" && + FLAGS_transport != "tcp") { + std::cerr << "Invalid transport\n"; + return 1; + } + const bool cooldown_ok = FLAGS_transport == "urma" ? + brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms) : + brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms); if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || - !brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms) || + !cooldown_ok || !SetMode(FLAGS_mode)) { std::cerr << "Invalid duration, interval, timeout, cooldown or mode\n"; return 1; } brpc::Channel channel; brpc::ChannelOptions options; - options.socket_mode = FLAGS_use_rdma ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA : + FLAGS_transport == "rdma" ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; options.connection_type = FLAGS_connection_type; options.timeout_ms = FLAGS_timeout_ms; options.connect_timeout_ms = FLAGS_timeout_ms; @@ -78,7 +94,7 @@ int main(int argc, char* argv[]) { uint64_t failed = 0; const int64_t start = butil::monotonic_time_us(); int64_t next_report = start; - std::cout << "seconds,mode,success,failure,rdma_negotiated,tcp_fallback_admissions," + std::cout << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," "cooled_peers,port_active,port_cooldown_ms\n"; while (!brpc::IsAskedToQuit() && butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { @@ -106,7 +122,8 @@ int main(int argc, char* argv[]) { } const int64_t now = butil::monotonic_time_us(); if (now >= next_report) { - const auto s = brpc::rdma::GetRdmaRecoveryStatus(); + const auto s = FLAGS_transport == "urma" ? + brpc::urma::GetUrmaRecoveryStatus() : brpc::rdma::GetRdmaRecoveryStatus(); std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded << ',' << failed << ',' << s.negotiated_connections << ',' << s.tcp_only_admissions + s.port_cooldown_admissions + diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp index fbaa4d6fd2..c8329a95a1 100644 --- a/src/brpc/rdma/rdma_recovery_policy.cpp +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -16,6 +16,7 @@ // under the License. #include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "bvar/passive_status.h" @@ -34,6 +35,8 @@ const char* RdmaFailureReasonName(RdmaFailureReason reason) { case RdmaFailureReason::CQ_NOTIFY: return "cq_notify"; case RdmaFailureReason::SEND_POST: return "send_post"; case RdmaFailureReason::ACCESS_ERROR: return "access_error"; + case RdmaFailureReason::TRANSPORT_ERROR: return "transport_error"; + case RdmaFailureReason::RECV_POST: return "recv_post"; default: return "unknown"; } } @@ -198,9 +201,10 @@ static void DescribeRecovery(std::ostream& os, void* arg) { } struct RecoveryMetrics { - explicit RecoveryMetrics(RdmaRecoveryPolicy* policy) - : status("rdma_recovery_status", DescribeRecovery, policy) - , fallbacks("rdma_recovery_tcp_fallback_admissions", [](void* arg) { + RecoveryMetrics(RdmaRecoveryPolicy* policy, const char* status_name, + const char* fallback_name) + : status(status_name, DescribeRecovery, policy) + , fallbacks(fallback_name, [](void* arg) { const auto s = static_cast(arg)->GetStatus(); return s.tcp_only_admissions + s.port_cooldown_admissions + s.peer_cooldown_admissions; @@ -214,7 +218,8 @@ RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { // Async event callbacks may outlive ordinary static destructors. static RdmaRecoveryPolicy* policy = new RdmaRecoveryPolicy(butil::monotonic_time_us); - static RecoveryMetrics* metrics = new RecoveryMetrics(policy); + static RecoveryMetrics* metrics = new RecoveryMetrics( + policy, "rdma_recovery_status", "rdma_recovery_tcp_fallback_admissions"); (void)metrics; return *policy; } @@ -236,4 +241,33 @@ RdmaRecoveryStatus GetRdmaRecoveryStatus() { } } // namespace rdma + +namespace urma { + +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy() { + static UrmaRecoveryPolicy* policy = + new UrmaRecoveryPolicy(butil::monotonic_time_us); + static rdma::RecoveryMetrics* metrics = new rdma::RecoveryMetrics( + policy, "urma_recovery_status", "urma_recovery_tcp_fallback_admissions"); + (void)metrics; + return *policy; +} + +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode) { + return GlobalUrmaRecoveryPolicy().SetMode(mode); +} + +UrmaRecoveryMode GetUrmaRecoveryMode() { + return GlobalUrmaRecoveryPolicy().GetMode(); +} + +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms) { + return GlobalUrmaRecoveryPolicy().SetCooldownMs(cooldown_ms); +} + +UrmaRecoveryStatus GetUrmaRecoveryStatus() { + return GlobalUrmaRecoveryPolicy().GetStatus(); +} + +} // namespace urma } // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h index 651cf8ccb5..c46c8a0705 100644 --- a/src/brpc/rdma/rdma_recovery_policy.h +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -31,7 +31,7 @@ enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; enum class RdmaFailureReason { NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, - CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, COUNT + CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, TRANSPORT_ERROR, RECV_POST, COUNT }; const char* RdmaFailureReasonName(RdmaFailureReason reason); diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 26c7a1b9dc..50e51bbf82 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -168,6 +168,8 @@ UrmaEndpoint::~UrmaEndpoint() { void UrmaEndpoint::Reset() { DeallocateResources(); _state = UNINIT; + _recovery_epoch = 0; + _runtime_failure.store(UrmaFailureReason::NONE, butil::memory_order_relaxed); _handshake_version = 0; _remote_recv_block_size = 0; _local_window_capacity = 0; @@ -770,6 +772,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { << ", configured_max_sge=" << GetUrmaMaxSge() << ", payload_size=" << this_len << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = rc; return -1; } @@ -815,6 +818,7 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { _resource->jetty->remote_jetty) << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::RECV_POST); errno = status; return -1; } @@ -908,6 +912,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { << _resource->remote_jetty->id.id << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = status; return -1; } @@ -928,6 +933,26 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { LOG(WARNING) << "URMA completion failed, status=" << cr.status; + switch (cr.status) { + case URMA_CR_ACK_TIMEOUT_ERR: + ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); + break; + case URMA_CR_RNR_RETRY_CNT_EXC_ERR: + ReportFailure(UrmaFailureReason::RNR); + break; + case URMA_CR_LOC_ACCESS_ERR: + case URMA_CR_REM_ACCESS_ABORT_ERR: + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + break; + case URMA_CR_WR_FLUSH_ERR: + case URMA_CR_WR_FLUSH_ERR_DONE: + case URMA_CR_WR_SUSPEND_DONE: + case URMA_CR_WR_UNHANDLED: + break; + default: + ReportFailure(UrmaFailureReason::TRANSPORT_ERROR); + break; + } errno = EIO; return -1; } @@ -946,8 +971,7 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { return -1; } _sq_imm_window_size += 1; - SendAck(0); - return 0; + return SendAck(0); } uint16_t wnd = 1; // We signal every WR (complete_enable=1). uint16_t old = @@ -1035,8 +1059,8 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (cr.completion_len > 0) { - SendAck(1); + if (cr.completion_len > 0 && SendAck(1) < 0) { + return -1; } return static_cast(cr.completion_len); } @@ -1118,6 +1142,7 @@ void UrmaEndpoint::PollCq(Socket* m) { const int cnt = urma_poll_jfc(ep->_resource->jfc, n, crs); if (cnt < 0) { + ep->ReportFailure(UrmaFailureReason::CQ_POLL); return EIO; } if (cnt == 0) { @@ -1307,6 +1332,21 @@ void UrmaEndpoint::FallbackToTcp(UrmaTransport* transport, bool process_tcp) { } } +void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { + if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { + return; + } + UrmaFailureReason expected = UrmaFailureReason::NONE; + if (reason == UrmaFailureReason::NONE || + !_runtime_failure.compare_exchange_strong(expected, reason)) { + return; + } + GlobalUrmaRecoveryPolicy().ReportPeerFailure( + _socket->CreatedByConnect() ? + butil::endpoint2str(_socket->remote_side()).c_str() : "", + _recovery_epoch, reason); +} + void UrmaEndpoint::FailHandshake(UrmaTransport* transport, int error, const char* reason) { LOG(ERROR) << "URMA handshake failed in state=" << GetStateStr() @@ -1333,7 +1373,9 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { SocketUniquePtr s(ep->_socket); auto* tp = static_cast(s->_transport.get()); UrmaConnect::RunGuard guard(static_cast(s->_app_connect.get())); - if (!IsUrmaAvailable()) { + if (!IsUrmaAvailable() || + !GlobalUrmaRecoveryPolicy().AllowNewConnection( + butil::endpoint2str(s->remote_side()).c_str(), &ep->_recovery_epoch)) { ep->FallbackToTcp(tp, true); return nullptr; } @@ -1385,6 +1427,7 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); return nullptr; } @@ -1420,6 +1463,16 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->FailHandshake(tp, EPROTO, "invalid client hello"); return nullptr; } + if (!IsUrmaAvailable() || + !GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch)) { + tp->_urma_state = UrmaTransport::URMA_OFF; + if (hs->SendLocalHello() < 0) { + ep->FailHandshake(tp, errno ? errno : EIO, "send TCP fallback hello"); + return nullptr; + } + ep->FallbackToTcp(tp, true); + return nullptr; + } ep->_state = S_ALLOC_RES; if (ep->AllocateResources() < 0) { const int saved_errno = errno ? errno : EIO; @@ -1470,6 +1523,7 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); } else { ep->FallbackToTcp(tp, true); @@ -1560,6 +1614,8 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " remote_recv_block_size=" << _remote_recv_block_size << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " recovery_epoch=" << _recovery_epoch + << " runtime_failure=" << rdma::RdmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } @@ -1579,6 +1635,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, const int saved_errno = errno; PLOG(ERROR) << "Fail to wait URMA JFC event from " << s->description(); + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to wait URMA JFC event: %s", berror(saved_errno)); return -1; @@ -1589,6 +1646,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, if (*event_jfc != _resource->jfc) { LOG(ERROR) << "Unexpected URMA JFC event on " << s->description(); errno = EPROTO; + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(EPROTO, "Unexpected URMA JFC event"); return -1; } @@ -1604,6 +1662,7 @@ int UrmaEndpoint::ReqNotifyCq() { if (rc != URMA_SUCCESS) { errno = rc; PLOG(WARNING) << "Fail to rearm URMA JFC"; + ReportFailure(UrmaFailureReason::CQ_NOTIFY); _socket->SetFailed(rc, "Fail to rearm URMA JFC: %s", berror(rc)); return -1; } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index e5aed5ac66..5d2a3908b0 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -18,6 +18,8 @@ #ifndef BRPC_URMA_ENDPOINT_H #define BRPC_URMA_ENDPOINT_H +#include "brpc/urma/urma_recovery_policy.h" + #include #include #include @@ -100,6 +102,7 @@ struct UrmaResource { // windows, and the registered buffer bookkeeping. Cache-line padded to avoid // false sharing between connections. class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { + friend class UrmaRecoveryTestPeer; friend class UrmaConnect; friend class Socket; friend class UrmaTransport; @@ -261,6 +264,7 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { inline void TryReadOnTcp(); void FallbackToTcp(UrmaTransport* transport, bool process_tcp); + void ReportFailure(UrmaFailureReason reason); void FailHandshake(UrmaTransport* transport, int error, const char* reason); @@ -272,6 +276,9 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Not owned. Socket* _socket; + uint64_t _recovery_epoch{0}; + butil::atomic _runtime_failure{UrmaFailureReason::NONE}; + // Handshake state. butil::atomic _state{UNINIT}; int _handshake_version{0}; // 0 = unnegotiated; 2 = v2; 3 = v3 diff --git a/src/brpc/urma/urma_handshake.cpp b/src/brpc/urma/urma_handshake.cpp index b7018606e0..ac7eb115ce 100644 --- a/src/brpc/urma/urma_handshake.cpp +++ b/src/brpc/urma/urma_handshake.cpp @@ -328,7 +328,12 @@ int UrmaHandshakeServerV3::ReceiveAndParseRemoteHello(ParsedHello* out, int UrmaHandshakeServerV3::SendLocalHello() { UrmaHello msg; _ep->FillLocalHelloV3(&msg); - // v3 has no zero-out path; the client rejects jetty_id==0 via ValidHello. + auto* tp = static_cast(_ep->_socket->_transport.get()); + if (tp->_urma_state.load(butil::memory_order_acquire) == + UrmaTransport::URMA_OFF) { + msg.set_jetty_id(0); + msg.set_buffer_size(0); + } return _ep->WriteHelloV3(msg); } diff --git a/src/brpc/urma/urma_recovery_policy.h b/src/brpc/urma/urma_recovery_policy.h new file mode 100644 index 0000000000..da2792fd54 --- /dev/null +++ b/src/brpc/urma/urma_recovery_policy.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_RECOVERY_POLICY_H +#define BRPC_URMA_RECOVERY_POLICY_H + +#include "brpc/rdma/rdma_recovery_policy.h" + +namespace brpc { +namespace urma { + +// Share the admission algorithm while keeping transport state independent. +using UrmaRecoveryMode = rdma::RdmaRecoveryMode; +using UrmaFailureReason = rdma::RdmaFailureReason; +using UrmaRecoveryStatus = rdma::RdmaRecoveryStatus; +using UrmaRecoveryPolicy = rdma::RdmaRecoveryPolicy; + +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy(); +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode); +UrmaRecoveryMode GetUrmaRecoveryMode(); +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms); +UrmaRecoveryStatus GetUrmaRecoveryStatus(); + +} // namespace urma +} // namespace brpc + +#endif // BRPC_URMA_RECOVERY_POLICY_H diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index d27367a504..21b76dbb6a 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include #include #include @@ -28,6 +30,15 @@ #include "brpc/urma/urma_handshake.pb.h" #include "brpc/urma/urma_helper.h" #include "urma_types.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/fd_guard.h" +#include "bvar/variable.h" +#include "echo.pb.h" using namespace brpc; @@ -593,6 +604,331 @@ TEST_F(UrmaMockTest, urma_free_device_list(devices); } + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static void Establish(UrmaEndpoint* ep) { + GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); + ep->_state = UrmaEndpoint::ESTABLISHED; + } + static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { + urma_cr_t cr{}; + cr.status = status; + return ep->HandleCompletion(cr); + } + static void Report(UrmaEndpoint* ep, UrmaFailureReason reason) { + ep->ReportFailure(reason); + } + static void PollFailure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jfc_t invalid_jfc{}; + resource.jfc = &invalid_jfc; + ep->_resource = &resource; + SocketOptions options; + options.user = ep; + SocketId sid; + if (Socket::Create(options, &sid) == 0) { + SocketUniquePtr carrier; + if (Socket::Address(sid, &carrier) == 0) { + const bool polling = FLAGS_urma_use_polling; + FLAGS_urma_use_polling = true; + UrmaEndpoint::PollCq(carrier.get()); + FLAGS_urma_use_polling = polling; + carrier->SetFailed(); + } + } + ep->_resource = nullptr; + resource.jfc = nullptr; + } + static bool Backpressure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_window_size = 0; + ep->_remote_rq_window_size = 1; + butil::IOBuf data; + data.append("payload"); + butil::IOBuf* buf = &data; + const ssize_t rc = ep->CutFromIOBufList(&buf, 1); + const bool ok = rc == -1 && errno == EAGAIN && data.size() == 7; + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return ok; + } + static int FailedAck(UrmaEndpoint* ep) { + // A Jetty without an imported target fails provider submission. + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_imm_window_size = 1; + ep->_remote_window_capacity = 2; + ep->_new_rq_wrs = 2; + urma_cr_t cr{}; + cr.status = URMA_CR_SUCCESS; + cr.user_ctx = 0; + const int rc = ep->HandleCompletion(cr); + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return rc; + } +}; + +} // namespace urma +} // namespace brpc + +class UrmaRecoveryTest : public UrmaMockTest { +protected: + void SetUp() override { + UrmaMockTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + urma::g_skip_urma_init = true; + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + urma::GlobalUrmaRecoveryPolicy().ReportPortState(true); + } + void TearDown() override { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + UrmaMockTest::TearDown(); + } + void CreateSocket(SocketUniquePtr* socket) { + SocketOptions opt; + butil::str2endpoint("127.0.0.1:8123", &opt.remote_side); + opt.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(opt, &sid)); + ASSERT_EQ(0, Socket::Address(sid, socket)); + } +}; + +TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { + ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_TRUE(rdma::GlobalRdmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_EQ(rdma::RdmaRecoveryMode::LEGACY, rdma::GetRdmaRecoveryMode()); + EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); + EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); + EXPECT_NE(std::string::npos, + bvar::Variable::describe_exposed("urma_recovery_status").find("tcp_only")); + EXPECT_FALSE(bvar::Variable::describe_exposed( + "urma_recovery_tcp_fallback_admissions").empty()); +} + +TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { + for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, + URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); + const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR ? + urma::UrmaFailureReason::RETRY_EXHAUSTED : + status == URMA_CR_RNR_RETRY_CNT_EXC_ERR ? urma::UrmaFailureReason::RNR : + status == URMA_CR_LOC_ACCESS_ERR ? urma::UrmaFailureReason::ACCESS_ERROR : + urma::UrmaFailureReason::TRANSPORT_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + urma::GetUrmaRecoveryStatus().failures[static_cast(reason)]); + EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)], + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); + socket->SetFailed(); + } +} + +TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + urma::UrmaRecoveryTestPeer::PollFailure(&ep); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + 1, + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); +} + +class UrmaRecoveryEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, + ::test::EchoResponse* response, google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + UrmaRecoveryEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + Channel channel; + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 2000; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + ::test::EchoService_Stub stub(&channel); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("recovery over TCP"); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketOptions failed_options; + failed_options.remote_side = server.listen_address(); + failed_options.user = get_client_side_messenger(); + SocketId failed_sid; + ASSERT_EQ(0, Socket::Create(failed_options, &failed_sid)); + SocketUniquePtr failed_socket; + ASSERT_EQ(0, Socket::Address(failed_sid, &failed_socket)); + { + urma::UrmaEndpoint failed_ep(failed_socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&failed_ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( + &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); + } + failed_socket->SetFailed(); + const auto before = urma::GetUrmaRecoveryStatus(); + cntl.Reset(); + response.Clear(); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(before.peer_cooldown_admissions + 1, + urma::GetUrmaRecoveryStatus().peer_cooldown_admissions); + server.Stop(0); + server.Join(); +} + +TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + Server server; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + for (int version : {2, 3}) { + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr)); + ASSERT_GE(fd, 0); + timeval timeout{2, 0}; + ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); + urma::v2_wire::HelloMessage hello; + hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; + hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; + hello.impl_ver = urma::v2_wire::IMPL_V2_VERSION; + hello.buffer_size = 8192; + hello.recv_buffer_cnt = 127; + hello.jetty_id = 42; + memset(hello.eid, 1, 16); + memset(hello.seg_eid, 1, 16); + hello.tp_type = URMA_CTP; + hello.seg_va = 0x100000; + hello.seg_len = 1024 * 1024; + if (version == 2) { + uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; + memcpy(packet, "URMA", 4); + hello.Serialize(packet + 4); + ASSERT_EQ(sizeof(packet), static_cast(write(fd, packet, sizeof(packet)))); + uint8_t reply[sizeof(packet)]; + ASSERT_EQ(sizeof(reply), static_cast(recv(fd, reply, sizeof(reply), MSG_WAITALL))); + EXPECT_EQ(0, memcmp(reply, "URMA", 4)); + urma::v2_wire::HelloMessage rejected; + rejected.Deserialize(reply + 4); + EXPECT_EQ(0, rejected.hello_ver); + EXPECT_EQ(0u, rejected.jetty_id); + } else { + urma::UrmaHello msg; + msg.set_buffer_size(8192); + msg.set_recv_buffer_cnt(127); + msg.set_jetty_id(42); + msg.set_uasid(0); + msg.set_seg_uasid(0); + msg.set_seg_token_id(0); + msg.set_eid(hello.eid, 16); + msg.set_seg_eid(hello.seg_eid, 16); + msg.set_tp_type(URMA_CTP); + msg.set_seg_va(hello.seg_va); + msg.set_seg_len(hello.seg_len); + const std::string body = msg.SerializeAsString(); + uint32_t size = butil::HostToNet32(body.size()); + std::string packet("URM3", 4); + packet.append(reinterpret_cast(&size), 4); + packet.append(body); + ASSERT_EQ(packet.size(), static_cast(write(fd, packet.data(), packet.size()))); + char header[8]; + ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); + EXPECT_EQ(0, memcmp(header, "URM3", 4)); + memcpy(&size, header + 4, 4); + size = butil::NetToHost32(size); + ASSERT_LE(size, 4096u); + std::string reply(size, '\0'); + ASSERT_EQ(size, static_cast(recv(fd, &reply[0], size, MSG_WAITALL))); + urma::UrmaHello rejected; + ASSERT_TRUE(rejected.ParseFromString(reply)); + EXPECT_EQ(0u, rejected.jetty_id()); + } + const std::string request = "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), request.size()))); + char response[4096]; + const ssize_t nr = recv(fd, response, sizeof(response), 0); + ASSERT_GT(nr, 0); + EXPECT_NE(std::string::npos, std::string(response, nr).find(" 200 ")); + } + server.Stop(0); + server.Join(); +} + #else // BRPC_WITH_URMA // When URMA is not compiled in, the test file is a no-op so the build stays From b0414f57cbf156a5dfd91cb0140006aea2c521eb Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 20:10:32 -0700 Subject: [PATCH 07/29] Adapt business QoS configuration to URMA --- docs/cn/urma.md | 28 +++ docs/en/urma.md | 31 ++++ example/urma_performance/client.cpp | 2 + example/urma_performance/server.cpp | 2 + src/brpc/acceptor.cpp | 1 + src/brpc/acceptor.h | 2 + src/brpc/channel.cpp | 7 + src/brpc/channel.h | 4 + src/brpc/server.cpp | 6 + src/brpc/server.h | 4 + src/brpc/socket.cpp | 7 + src/brpc/socket.h | 4 + src/brpc/urma/mock_urma.cpp | 3 + src/brpc/urma/urma_endpoint.cpp | 15 +- src/brpc/urma/urma_endpoint.h | 1 + src/brpc/urma/urma_helper.cpp | 25 +++ src/brpc/urma/urma_helper.h | 6 + src/brpc/urma/urma_qos.h | 32 ++++ src/brpc/urma_transport.cpp | 10 ++ test/brpc_urma_qos_unittest.cpp | 256 ++++++++++++++++++++++++++++ 20 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 src/brpc/urma/urma_qos.h create mode 100644 test/brpc_urma_qos_unittest.cpp diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 62e10389a8..14105a02d1 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -193,3 +193,31 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing `ContextInitOrDie` 阶段拒绝。 - 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 - 当前实现面向 Linux。 + +## 业务优先级 + +`ChannelOptions.urma_qos.priority` 控制请求发送方向, +`ServerOptions.urma_qos.priority` 控制响应发送方向: + +```cpp +brpc::ChannelOptions client_options; +client_options.socket_mode = brpc::SOCKET_MODE_URMA; +client_options.urma_qos.priority = 10; + +brpc::ServerOptions server_options; +server_options.socket_mode = brpc::SOCKET_MODE_URMA; +server_options.urma_qos.priority = 6; +``` + +默认值 -1 沿用设备自动选择,取设备能力表中首个 CTP priority;未报告时沿用兼容值 6。 +显式值范围为 0–15,并要求所选设备的 `priority_info[priority].tp_type` 包含 CTP。 +不满足范围或能力要求时,Channel 初始化和 Server 启动返回错误。 +priority 对应的 SL 和调度待遇由设备及网络配置决定。 + +不同 priority 使用独立连接和 Jetty,相同配置共享连接。配置继承到服务发现、连接池、 +短连接和服务端 accept 路径,重建时重新应用。默认预建池只用于匹配其 priority 的连接, +其他 priority 按需创建 Jetty。连接调试信息显示配置值和实际 Jetty priority。 +请求和响应分别配置各自的发送策略,v2/v3 握手格式沿用现有协议。 + +`example/urma_performance` 的 client/server 支持 `--urma_priority=10`。 +以不同 priority 启动竞争流量,对比吞吐和 P99,并结合设备 SL 映射验证调度效果。 diff --git a/docs/en/urma.md b/docs/en/urma.md index a5b80f45b9..caa5eb09b3 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -207,3 +207,34 @@ is tracked as a future enhancement. rejected at `ContextInitOrDie`. - A hardware data path requires a supported UMDK provider and `liburma`. - The current implementation targets Linux. + +## Business priority + +Set `ChannelOptions.urma_qos.priority` for outgoing requests and +`ServerOptions.urma_qos.priority` for outgoing responses: + +```cpp +brpc::ChannelOptions client_options; +client_options.socket_mode = brpc::SOCKET_MODE_URMA; +client_options.urma_qos.priority = 10; + +brpc::ServerOptions server_options; +server_options.socket_mode = brpc::SOCKET_MODE_URMA; +server_options.urma_qos.priority = 6; +``` + +The default -1 keeps automatic selection: the first CTP priority reported by the +device, with the existing compatibility value 6 when none is reported. Explicit +values are in [0,15] and require CTP in `priority_info[priority].tp_type` on the +selected device. Channel initialization and Server startup reject unsupported +values. Device and network configuration determine SL mapping and scheduling. + +Different priorities use separate connections and Jettys; equal configurations +share connections. Naming services, pooled and short connections, accepted sockets +and rebuilt Jettys inherit the configuration. The prepared pool serves matching +priorities; other priorities allocate Jettys on demand. Debug output shows both +the configured and effective priority. Each peer configures its outgoing direction; +v2/v3 handshake formats remain the same. + +The client and server in `example/urma_performance` accept `--urma_priority=10`. +Compare throughput and P99 under competing priorities using the device SL mapping. diff --git a/example/urma_performance/client.cpp b/example/urma_performance/client.cpp index 9449dbe960..a66043b2d7 100644 --- a/example/urma_performance/client.cpp +++ b/example/urma_performance/client.cpp @@ -44,6 +44,7 @@ DEFINE_int32(attachment_size, -1, "Attachment size is used (in Bytes)"); DEFINE_int32(rpc_timeout_ms, 5000, "Timeout for each RPC in milliseconds"); DEFINE_bool(echo_attachment, false, "Select whether attachment should be echo"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); +DEFINE_int32(urma_priority, -1, "Outbound URMA priority: -1 automatic, or [0,15]"); bvar::LatencyRecorder g_latency("client"); bvar::Adder g_error_count("client_error_count"); @@ -88,6 +89,7 @@ static void* worker(void* arg) { int main(int argc, char* argv[]) { gflags::ParseCommandLineFlags(&argc, &argv, true); brpc::ChannelOptions options; + options.urma_qos.priority = FLAGS_urma_priority; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; options.connect_timeout_ms = FLAGS_rpc_timeout_ms; diff --git a/example/urma_performance/server.cpp b/example/urma_performance/server.cpp index baaff323b6..b874b6a424 100644 --- a/example/urma_performance/server.cpp +++ b/example/urma_performance/server.cpp @@ -30,6 +30,7 @@ DEFINE_int32(port, 8003, "TCP Port of this server"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); +DEFINE_int32(urma_priority, -1, "Outbound URMA priority: -1 automatic, or [0,15]"); butil::atomic g_last_time(0); @@ -73,6 +74,7 @@ int main(int argc, char* argv[]) { } brpc::ServerOptions options; + options.urma_qos.priority = FLAGS_urma_priority; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; if (server.Start(FLAGS_port, &options) != 0) { diff --git a/src/brpc/acceptor.cpp b/src/brpc/acceptor.cpp index d19d98ee0a..9c5504f207 100644 --- a/src/brpc/acceptor.cpp +++ b/src/brpc/acceptor.cpp @@ -339,6 +339,7 @@ void Acceptor::OnNewConnectionsUntilEAGAIN(Socket* acception) { options.initial_ssl_ctx = am->_ssl_ctx; options.socket_mode = am->_socket_mode; options.rdma_qos = am->_rdma_qos; + options.urma_qos = am->_urma_qos; options.bthread_tag = am->_bthread_tag; if (Socket::Create(options, &socket_id) != 0) { am->ReleaseConnectionSlot(); diff --git a/src/brpc/acceptor.h b/src/brpc/acceptor.h index 2485d9f387..b474fdd5a0 100644 --- a/src/brpc/acceptor.h +++ b/src/brpc/acceptor.h @@ -25,6 +25,7 @@ #include "brpc/input_messenger.h" #include "brpc/socket_mode.h" #include "brpc/rdma/rdma_qos.h" +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -139,6 +140,7 @@ friend class Server; // Choose to use a certain socket: 0 TCP, 1 RDMA SocketMode _socket_mode; RdmaQosOptions _rdma_qos; + UrmaQosOptions _urma_qos; // Acceptor belongs to this tag bthread_tag_t _bthread_tag; diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index 9a7811f8f3..ea87d58e7e 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -144,6 +144,7 @@ static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { buf.append("|tc=").append(std::to_string(opt.rdma_qos.traffic_class)); } else if (opt.socket_mode == SOCKET_MODE_URMA) { buf.append("|urma"); + buf.append("|priority=").append(std::to_string(opt.urma_qos.priority)); } butil::MurmurHash3_x64_128_Update(&mm_ctx, buf.data(), buf.size()); buf.clear(); @@ -194,6 +195,10 @@ int Channel::InitChannelOptions(const ChannelOptions* options) { LOG(ERROR) << "RDMA SL must be in [0,15] and traffic class in [0,255]"; return -1; } + if (_options.socket_mode == SOCKET_MODE_URMA && !_options.urma_qos.IsValid()) { + LOG(ERROR) << "URMA priority must be -1 (automatic) or in [0,15]"; + return -1; + } const Protocol* protocol = FindProtocol(_options.protocol); if (nullptr == protocol || !protocol->support_client()) { LOG(ERROR) << "Channel does not support the protocol"; @@ -415,6 +420,7 @@ int Channel::InitSingle(const butil::EndPoint& server_addr_and_port, opt.initial_ssl_ctx = ssl_ctx; opt.socket_mode = _options.socket_mode; opt.rdma_qos = _options.rdma_qos; + opt.urma_qos = _options.urma_qos; opt.hc_option = _options.hc_option; opt.device_name = _options.device_name; if (SocketMapInsert(SocketMapKey(server_addr_and_port, sig), @@ -462,6 +468,7 @@ int Channel::Init(const char* ns_url, ns_opt.log_succeed_without_server = _options.log_succeed_without_server; ns_opt.socket_option.socket_mode = _options.socket_mode; ns_opt.socket_option.rdma_qos = _options.rdma_qos; + ns_opt.socket_option.urma_qos = _options.urma_qos; ns_opt.channel_signature = ComputeChannelSignature(_options); ns_opt.socket_option.hc_option = _options.hc_option; ns_opt.socket_option.local_side = client_endpoint; diff --git a/src/brpc/channel.h b/src/brpc/channel.h index b4bff6092b..f11bdb04d7 100644 --- a/src/brpc/channel.h +++ b/src/brpc/channel.h @@ -39,6 +39,7 @@ #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" #include "brpc/rdma/rdma_qos.h" +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -115,6 +116,9 @@ struct ChannelOptions { // Outbound RDMA QoS for this Channel. Used when socket_mode is RDMA. RdmaQosOptions rdma_qos; + // Outbound URMA QoS for this Channel. Used when socket_mode is URMA. + UrmaQosOptions urma_qos; + // Turn on authentication for this channel if `auth' is not nullptr. // Note `auth' will not be deleted by channel and must remain valid when // the channel is being used. diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 96734ae8d0..4ed8bba4ae 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -880,6 +880,11 @@ int Server::StartInternal(const butil::EndPoint& endpoint, return -1; } + if (real_opt.socket_mode == SOCKET_MODE_URMA && !real_opt.urma_qos.IsValid()) { + LOG(ERROR) << "URMA priority must be -1 (automatic) or in [0,15]"; + return -1; + } + if (!real_opt.h2_settings.IsValid(true/*log_error*/)) { LOG(ERROR) << "Invalid h2_settings"; return -1; @@ -1168,6 +1173,7 @@ int Server::StartInternal(const butil::EndPoint& endpoint, _am->_bthread_tag = _options.bthread_tag; } _am->_rdma_qos = _options.rdma_qos; + _am->_urma_qos = _options.urma_qos; // Set `_status' to RUNNING before accepting connections // to prevent requests being rejected as ELOGOFF _status = RUNNING; diff --git a/src/brpc/server.h b/src/brpc/server.h index 769cee4150..83e1d6027d 100644 --- a/src/brpc/server.h +++ b/src/brpc/server.h @@ -47,6 +47,7 @@ #include "brpc/rpc_pb_message_factory.h" #include "brpc/socket_mode.h" #include "brpc/rdma/rdma_qos.h" +#include "brpc/urma/urma_qos.h" namespace brpc { @@ -246,6 +247,9 @@ struct ServerOptions { // Outbound RDMA QoS for responses on this server's accepted connections. RdmaQosOptions rdma_qos; + // Outbound URMA QoS for responses on accepted connections. + UrmaQosOptions urma_qos; + // [CAUTION] This option is for implementing specialized baidu-std proxies, // most users don't need it. Don't change this option unless you fully // understand the description below. diff --git a/src/brpc/socket.cpp b/src/brpc/socket.cpp index f560320b4b..4b49d9ea72 100644 --- a/src/brpc/socket.cpp +++ b/src/brpc/socket.cpp @@ -720,6 +720,10 @@ int Socket::Create(const SocketOptions& options, SocketId* id) { errno = EINVAL; return -1; } + if (options.socket_mode == SOCKET_MODE_URMA && !options.urma_qos.IsValid()) { + errno = EINVAL; + return -1; + } return VersionedRefWithId::Create(id, options); } @@ -736,6 +740,7 @@ int Socket::OnCreated(const SocketOptions& options) { // start build the transport _socket_mode = options.socket_mode; _rdma_qos = options.rdma_qos; + _urma_qos = options.urma_qos; _transport = TransportFactory::CreateTransport(options.socket_mode); CHECK(nullptr != _transport); _transport->Init(this, options); @@ -2836,6 +2841,7 @@ int Socket::GetPooledSocket(SocketUniquePtr* pooled_socket) { opt.app_connect = _app_connect; opt.socket_mode = _socket_mode; opt.rdma_qos = _rdma_qos; + opt.urma_qos = _urma_qos; socket_pool = new SocketPool(opt); SocketPool* expected = nullptr; if (!main_sp->socket_pool.compare_exchange_strong( @@ -2943,6 +2949,7 @@ int Socket::GetShortSocket(SocketUniquePtr* short_socket) { opt.app_connect = _app_connect; opt.socket_mode = _socket_mode; opt.rdma_qos = _rdma_qos; + opt.urma_qos = _urma_qos; if (get_client_side_messenger()->Create(opt, &id) != 0 || Address(id, short_socket) != 0) { return -1; diff --git a/src/brpc/socket.h b/src/brpc/socket.h index 9cd26674c4..8911b801a8 100644 --- a/src/brpc/socket.h +++ b/src/brpc/socket.h @@ -44,6 +44,7 @@ #include "brpc/health_check_option.h" #include "brpc/socket_mode.h" #include "brpc/rdma/rdma_qos.h" +#include "brpc/urma/urma_qos.h" #include "brpc/input_messenger_processor.h" // InputMessengerProcessor namespace brpc { @@ -311,6 +312,7 @@ struct SocketOptions { std::shared_ptr initial_ssl_ctx; SocketMode socket_mode{SOCKET_MODE_TCP}; RdmaQosOptions rdma_qos; + UrmaQosOptions urma_qos; bthread_keytable_pool_t* keytable_pool{nullptr}; SocketConnection* conn{nullptr}; std::shared_ptr app_connect; @@ -635,6 +637,7 @@ friend class TransportFactory; // The socket mode decided at creation. SocketMode socket_mode() const { return _socket_mode; } const RdmaQosOptions& rdma_qos() const { return _rdma_qos; } + const UrmaQosOptions& urma_qos() const { return _urma_qos; } // Get an UNUSED socket connecting to the same place as this socket // from the SocketPool of this socket. @@ -1003,6 +1006,7 @@ friend class TransportFactory; // Should use SOCKET_MODE_RDMA or SOCKET_MODE_TCP or Other, default is SOCKET_MODE_TCP Transport SocketMode _socket_mode; RdmaQosOptions _rdma_qos; + UrmaQosOptions _urma_qos; std::unique_ptr _transport; // Pass from controller, for progressive reading. diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index b0572c2e86..f81b54bf1a 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -240,6 +240,9 @@ urma_status_t urma_query_device(urma_device_t *device, if (!device || !attr) { return URMA_EINVAL; } + mock_device_attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; mock_device_attr.dev_cap.max_jfc = 1024; mock_device_attr.dev_cap.max_jetty = 1024; mock_device_attr.dev_cap.max_jfs_sge = 8; diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 26c7a1b9dc..c3fc26e1af 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -390,6 +390,10 @@ int UrmaEndpoint::AllocateResources() { if (_resource) { return 0; } + uint8_t priority; + if (!ResolveUrmaJettyPriority(_socket->urma_qos(), &priority)) { + return -1; + } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { errno = ENODEV; @@ -401,8 +405,9 @@ int UrmaEndpoint::AllocateResources() { return -1; } - // Try the prepared pool first (sized sq/rq match). - if (_sq_size <= static_cast(FLAGS_urma_sq_size) && + // Prepared Jettys carry the default priority; other priorities allocate on demand. + if (priority == GetUrmaJettyPriority() && + _sq_size <= static_cast(FLAGS_urma_sq_size) && _rq_size <= static_cast(FLAGS_urma_rq_size)) { BAIDU_SCOPED_LOCK(g_prepared_mutex); if (g_prepared_list) { @@ -451,7 +456,7 @@ int UrmaEndpoint::AllocateResources() { jetty_cfg.flag.bs.share_jfr = 1; jetty_cfg.jfs_cfg.depth = static_cast(_sq_size); jetty_cfg.jfs_cfg.trans_mode = URMA_TM_RM; - jetty_cfg.jfs_cfg.priority = GetUrmaJettyPriority(); + jetty_cfg.jfs_cfg.priority = priority; jetty_cfg.jfs_cfg.max_sge = static_cast(GetUrmaMaxSge()); jetty_cfg.jfs_cfg.rnr_retry = URMA_TYPICAL_RNR_RETRY; @@ -1560,6 +1565,10 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " remote_recv_block_size=" << _remote_recv_block_size << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " urma_priority=" << _socket->urma_qos().priority + << " urma_effective_priority=" + << (_resource && _resource->jetty ? + static_cast(_resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index e5aed5ac66..a237758f2c 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -100,6 +100,7 @@ struct UrmaResource { // windows, and the registered buffer bookkeeping. Cache-line padded to avoid // false sharing between connections. class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { + friend class UrmaQosTestPeer; friend class UrmaConnect; friend class Socket; friend class UrmaTransport; diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index ded662fb22..e15bef6494 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -687,6 +687,31 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, return -1; } uint8_t GetUrmaJettyPriority() { return g_jetty_priority; } + +bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority) { + return priority >= 0 && priority <= URMA_MAX_PRIORITY && + attr.dev_cap.priority_info[priority].tp_type.bs.ctp; +} + +bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { + if (!qos.IsValid()) { + errno = EINVAL; + return false; + } + if (qos.priority == -1) { + *priority = GetUrmaJettyPriority(); + return true; + } + if (!IsUrmaAvailable() || !g_context || + !IsUrmaCtpPrioritySupported(g_device_attr, qos.priority)) { + LOG(ERROR) << "URMA priority " << qos.priority + << " does not advertise CTP on the selected device"; + errno = EINVAL; + return false; + } + *priority = static_cast(qos.priority); + return true; +} int GetUrmaMaxSge() { return g_max_sge; } size_t GetUrmaRecvBlockSize() { return g_recv_block_size; } diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 871f1a6354..de663e9906 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -29,6 +29,7 @@ #if BRPC_WITH_URMA #include "urma_api.h" +#include "brpc/urma/urma_qos.h" #include "urma_types.h" namespace brpc { @@ -83,6 +84,11 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, // Return the priority selected for the CTP jettys created by brpc. uint8_t GetUrmaJettyPriority(); +// Check whether an explicit priority advertises the CTP path used by brpc. +bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority); +// Resolve automatic selection or validate against the initialized device. +bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/src/brpc/urma/urma_qos.h b/src/brpc/urma/urma_qos.h new file mode 100644 index 0000000000..5c8924bdee --- /dev/null +++ b/src/brpc/urma/urma_qos.h @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_QOS_H +#define BRPC_URMA_QOS_H + +namespace brpc { + +// Outbound Jetty priority. Device CTP capabilities define valid selections. +struct UrmaQosOptions { + int priority = -1; // -1 selects the device default; explicit range [0, 15]. + + bool IsValid() const { return priority >= -1 && priority <= 15; } +}; + +} // namespace brpc + +#endif // BRPC_URMA_QOS_H diff --git a/src/brpc/urma_transport.cpp b/src/brpc/urma_transport.cpp index c5cc163ffa..a1e8f9b425 100644 --- a/src/brpc/urma_transport.cpp +++ b/src/brpc/urma_transport.cpp @@ -197,6 +197,11 @@ int UrmaTransport::ContextInitOrDie(bool server_or_not, const void* options) { return -1; } urma::GlobalUrmaInitializeOrDie(); + uint8_t priority; + if (!urma::ResolveUrmaJettyPriority( + static_cast(options)->urma_qos, &priority)) { + return -1; + } if (!urma::InitPollingModeWithTag( static_cast(options)->bthread_tag)) { return -1; @@ -207,6 +212,11 @@ int UrmaTransport::ContextInitOrDie(bool server_or_not, const void* options) { return -1; } urma::GlobalUrmaInitializeOrDie(); + uint8_t priority; + if (!urma::ResolveUrmaJettyPriority( + static_cast(options)->urma_qos, &priority)) { + return -1; + } if (!urma::InitPollingModeWithTag(bthread_self_tag())) { return -1; } diff --git a/test/brpc_urma_qos_unittest.cpp b/test/brpc_urma_qos_unittest.cpp new file mode 100644 index 0000000000..7e63bb5878 --- /dev/null +++ b/test/brpc_urma_qos_unittest.cpp @@ -0,0 +1,256 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/acceptor.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/load_balancer.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma_transport.h" +#include "echo.pb.h" + +using namespace brpc; + +namespace brpc { +namespace urma { +DECLARE_int32(urma_client_handshake_version); +extern bool g_skip_urma_init; +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); + +class UrmaQosTestPeer { +public: + static int Allocate(UrmaEndpoint* ep) { return ep->AllocateResources(); } + static int Priority(UrmaEndpoint* ep) { + return ep->_resource->jetty->jetty_cfg.jfs_cfg.priority; + } +}; + +} // namespace urma +} // namespace brpc + +TEST(UrmaHelperTest, explicit_priority_requires_ctp_capability) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 0)); + EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 15)); + for (int priority : {-2, -1, 3, 4, 16}) { + EXPECT_FALSE(urma::IsUrmaCtpPrioritySupported(attr, priority)); + } +} + +class UrmaQosTest : public ::testing::Test { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (!_mock) { + return; + } + urma::g_skip_urma_init = false; + urma::FLAGS_urma_buffer_count = 1024; + urma::FLAGS_urma_prepared_jetty_cnt = 2; + urma::GlobalUrmaInitializeOrDie(); + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "URMA QoS integration tests use the link-time mock"; + } + ASSERT_TRUE(urma::IsUrmaAvailable()); + ASSERT_NE(nullptr, urma::GetUrmaContext()); + } + static bool _mock; + static urma::UrmaEndpoint* Endpoint(SocketUniquePtr& socket) { + return static_cast(socket->_transport.get())->GetUrmaEp(); + } +}; +bool UrmaQosTest::_mock = false; + +TEST_F(UrmaQosTest, rejects_invalid_and_unsupported_priority) { + butil::EndPoint address; + ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &address)); + for (int priority : {-2, 16, 3, 9}) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = priority; + Channel channel; + EXPECT_EQ(-1, channel.Init(address, &co)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos = co.urma_qos; + Server server; + EXPECT_EQ(-1, server.Start("127.0.0.1:0", &so)); + if (!co.urma_qos.IsValid()) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + EXPECT_EQ(-1, Socket::Create(options, &sid)); + EXPECT_EQ(EINVAL, errno); + } else { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(EINVAL, errno); + socket->SetFailed(); + } + } +} + +TEST_F(UrmaQosTest, channels_isolate_priority_and_inherit_to_child_connections) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + Channel automatic, low, same, high, named, tcp, tcp_with_qos; + ASSERT_EQ(0, automatic.Init("127.0.0.1:8123", &co)); + co.urma_qos.priority = 6; + ASSERT_EQ(0, low.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, same.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, named.Init("list://127.0.0.1:8123", "rr", &co)); + co.urma_qos.priority = 10; + ASSERT_EQ(0, high.Init("127.0.0.1:8123", &co)); + EXPECT_EQ(low._server_id, same._server_id); + EXPECT_NE(low._server_id, high._server_id); + EXPECT_NE(low._server_id, automatic._server_id); + co.socket_mode = SOCKET_MODE_TCP; + ASSERT_EQ(0, tcp_with_qos.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, tcp.Init("127.0.0.1:8123", nullptr)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); + EXPECT_NE(tcp._server_id, automatic._server_id); + SocketUniquePtr main, pooled, short_socket, selected; + ASSERT_EQ(0, Socket::Address(low._server_id, &main)); + ASSERT_EQ(0, main->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (auto* socket : {main.get(), pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(6, socket->urma_qos().priority); + } + EXPECT_EQ(main->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); +} + +TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { + urma::UrmaEndpoint::GlobalRelease(); + ASSERT_EQ(0, urma::UrmaEndpoint::GlobalInitialize()); + // Allocate priority 10 before consuming the two prepared default Jettys. + for (int priority : {10, -1, 6, 10}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.priority = priority; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = Endpoint(socket); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); + EXPECT_EQ(priority == -1 ? 6 : priority, urma::UrmaQosTestPeer::Priority(ep)); + std::ostringstream debug; + ep->DebugInfo(debug); + EXPECT_NE(std::string::npos, debug.str().find("urma_effective_priority=")); + socket->SetFailed(); + } +} + +class UrmaQosEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, + ::test::EchoResponse* response, google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaQosTest, request_and_response_priorities_apply_across_restart) { + GFLAGS_NAMESPACE::FlagSaver saved_flags; + UrmaQosEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + for (int response_priority : {6, 10}) { + urma::FLAGS_urma_client_handshake_version = response_priority == 6 ? 2 : 3; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos.priority = response_priority; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = response_priority == 6 ? 10 : 6; + co.timeout_ms = 2000; + co.max_retry = 0; + co.connection_type = "single"; + Channel channel; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("URMA QoS"); + ::test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + SocketUniquePtr client; + ASSERT_EQ(0, Socket::Address(channel._server_id, &client)); + EXPECT_EQ(co.urma_qos.priority, urma::UrmaQosTestPeer::Priority(Endpoint(client))); + std::vector sockets; + server._am->ListConnections(&sockets); + ASSERT_EQ(1u, sockets.size()); + SocketUniquePtr accepted; + ASSERT_EQ(0, Socket::Address(sockets[0], &accepted)); + EXPECT_EQ(response_priority, accepted->urma_qos().priority); + EXPECT_EQ(response_priority, urma::UrmaQosTestPeer::Priority(Endpoint(accepted))); + accepted.reset(); + client.reset(); + server.Stop(0); + server.Join(); + } +} + +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} From c371849eb7bd01c4ae149f1fb74c14648d44faba Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:01:51 -0700 Subject: [PATCH 08/29] Add URMA memory access failure diagnostics --- src/brpc/urma/mock_urma.cpp | 8 +++++++- src/brpc/urma/urma_endpoint.cpp | 20 ++++++++++++++++++-- src/brpc/urma/urma_helper.cpp | 6 ++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index b0572c2e86..a1844d05e4 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -390,13 +390,19 @@ urma_target_seg_t *urma_register_seg(urma_context_t *ctx, urma_seg_cfg_t *cfg) { if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) { return nullptr; } - urma_target_seg_t *seg = new urma_target_seg_t; + urma_target_seg_t *seg = new urma_target_seg_t{}; memset(&seg->seg.ubva.eid, 0, sizeof(urma_eid_t)); seg->seg.ubva.eid.raw[0] = 1; seg->seg.ubva.uasid = 0; seg->seg.ubva.va = cfg->va; seg->seg.len = cfg->len; seg->seg.token_id = cfg->token_value.token; + seg->seg.attr.bs.access = cfg->flag.bs.access; + seg->seg.attr.bs.token_policy = cfg->flag.bs.token_policy; + seg->seg.attr.bs.cacheable = cfg->flag.bs.cacheable; + seg->user_ctx = cfg->user_ctx; + seg->urma_ctx = ctx; + seg->token_id = cfg->token_id; seg_map[seg] = 1; return seg; } diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 50e51bbf82..0b4771ac9b 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -800,7 +800,8 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { urma_sge_t sge{reinterpret_cast(block), static_cast(block_size), tseg, nullptr}; urma_sg_t sg{&sge, 1}; - urma_jfr_wr_t wr{sg, 0, nullptr}; + // Preserve the receive address in completions for access-error diagnostics. + urma_jfr_wr_t wr{sg, reinterpret_cast(block), nullptr}; urma_jfr_wr_t* bad = nullptr; // Use the shared-JFR path on every device, including bonding. The bonding // provider owns physical receive scheduling for the JFR; a local @@ -932,7 +933,22 @@ int UrmaEndpoint::SendAck(int num) { ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { - LOG(WARNING) << "URMA completion failed, status=" << cr.status; + LOG(WARNING) << "URMA completion failed, status=" << cr.status + << " direction=" << (cr.flag.bs.s_r ? "recv" : "send") + << " flags=" << static_cast(cr.flag.value) + << " opcode=" << cr.opcode + << " local_id=" << cr.local_id + << " tpn=" << cr.tpn + << " user_ctx=" << cr.user_ctx + << " completion_len=" << cr.completion_len + << " jetty_id=" + << (_resource && _resource->jetty ? + _resource->jetty->jetty_id.id : 0) + << " jfr_id=" + << (_resource && _resource->jfr ? + _resource->jfr->jfr_id.id : 0) + << " state=" << GetStateStr() + << " on " << _socket->description(); switch (cr.status) { case URMA_CR_ACK_TIMEOUT_ERR: ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index ded662fb22..d9dd06bc72 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -383,6 +383,12 @@ bool InitPool() { g_pool_base = nullptr; return false; } + LOG(INFO) << "URMA registered buffer pool: base=" << g_pool_base + << " size=" << g_pool_size + << " segment_va=" << g_pool_seg->seg.ubva.va + << " segment_len=" << g_pool_seg->seg.len + << " segment_access=" << g_pool_seg->seg.attr.bs.access + << " segment_token_id=" << g_pool_seg->seg.token_id; g_pool->in_use.assign(count, 0); for (size_t i = 0; i < count; ++i) { auto shard = i % kShardCount; From d87dfe891173fef7394f556395d10575fea325b2 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:58:49 -0700 Subject: [PATCH 09/29] Initialize and validate URMA send descriptors --- src/brpc/urma/urma_endpoint.cpp | 40 +++++++++++++++++++++++++++++---- src/brpc/urma/urma_helper.cpp | 21 +++++++++++++++++ src/brpc/urma/urma_helper.h | 4 ++++ test/brpc_urma_unittest.cpp | 38 +++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 0b4771ac9b..2545a69987 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -663,9 +663,16 @@ class UrmaIOBuf : private butil::IOBuf { if (len + this_len > max_len) { this_len = max_len - len; } - sglist[*sge_index].addr = reinterpret_cast(start); - sglist[*sge_index].len = static_cast(this_len); - sglist[*sge_index].tseg = tseg; + if (!InitUrmaLocalSge(&sglist[*sge_index], + reinterpret_cast(start), this_len, tseg)) { + LOG(ERROR) << "Invalid URMA send SGE: address=" + << reinterpret_cast(start) + << " length=" << this_len + << " segment_va=" << tseg->seg.ubva.va + << " segment_len=" << tseg->seg.len + << " context_matches=" << (tseg->urma_ctx == GetUrmaContext()); + return -1; + } cutn(to, this_len); len += this_len; (*sge_index)++; @@ -719,6 +726,9 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ssize_t n = data->cut_into_sglist(sglist, &sge_index, to, max_sge, max_len - this_len); if (n < 0) { + if (errno == ERDMAMEM) { + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + } return -1; } this_len += n; @@ -738,7 +748,8 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { wr.flag.bs.complete_enable = 1; wr.tjetty = _resource->remote_jetty; wr.send.src = sg; - wr.user_ctx = 1; + // Zero identifies credit ACKs; data completions identify their SQ slot. + wr.user_ctx = static_cast(_sq_current) + 1; urma_jfs_wr_t* bad_wr = nullptr; const uint16_t sq_slot = _sq_current; const uint32_t local_jetty_id = _resource->jetty->jetty_id.id; @@ -949,6 +960,27 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { _resource->jfr->jfr_id.id : 0) << " state=" << GetStateStr() << " on " << _socket->description(); + if (!cr.flag.bs.s_r && cr.user_ctx > 0 && cr.user_ctx <= _sbuf.size()) { + const size_t slot = static_cast(cr.user_ctx - 1); + const auto& buffer = _sbuf[slot]; + LOG(WARNING) << "URMA failed SEND buffer: slot=" << slot + << " bytes=" << buffer.size() + << " blocks=" << buffer.backing_block_num() + << " on " << _socket->description(); + for (size_t i = 0; i < buffer.backing_block_num(); ++i) { + const auto block = buffer.backing_block(i); + auto* segment = GetPoolSegFor(const_cast(block.data())); + LOG(WARNING) << "URMA failed SEND block: slot=" << slot + << " block=" << i + << " address=" << reinterpret_cast(block.data()) + << " length=" << block.size() + << " pool_segment=" << (segment != nullptr) + << " segment_va=" << (segment ? segment->seg.ubva.va : 0) + << " segment_len=" << (segment ? segment->seg.len : 0) + << " segment_token_id=" << (segment ? segment->seg.token_id : 0) + << " on " << _socket->description(); + } + } switch (cr.status) { case URMA_CR_ACK_TIMEOUT_ERR: ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index d9dd06bc72..f7c94d4827 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -418,6 +419,26 @@ urma_target_seg_t* GetPoolSegFor(void* buf) { return nullptr; } +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment) { + if (!sge || !segment || length == 0 || + length > std::numeric_limits::max() || + segment->urma_ctx != GetUrmaContext()) { + errno = ERDMAMEM; + return false; + } + const uint64_t base = segment->seg.ubva.va; + const uint64_t size = segment->seg.len; + if (size > std::numeric_limits::max() - base || + address < base || address - base > size || + length > size - (address - base)) { + errno = ERDMAMEM; + return false; + } + *sge = urma_sge_t{address, static_cast(length), segment, nullptr}; + return true; +} + static void GlobalRelease() { g_urma_available.store(false, butil::memory_order_release); if (g_mem_alloc_orig) { diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 871f1a6354..c7439812d7 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -83,6 +83,10 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, // Return the priority selected for the CTP jettys created by brpc. uint8_t GetUrmaJettyPriority(); +// Initialize a local SGE whose complete byte range belongs to this context. +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index 21b76dbb6a..da77a94d48 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -274,6 +274,44 @@ TEST(UrmaHelperTest, selects_priority_matching_transport_path_type) { EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); } +TEST(UrmaHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge; + memset(&sge, 0xff, sizeof(sge)); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x1000, 8192, &segment)); + EXPECT_EQ(0x1000u, sge.addr); + EXPECT_EQ(8192u, sge.len); + EXPECT_EQ(&segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x2fff, 1, &segment)); + EXPECT_EQ(0x2fffu, sge.addr); + EXPECT_EQ(1u, sge.len); +} + +TEST(UrmaHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge{}; + for (auto values : {std::pair(0xfff, 1), + {0x3000, 1}, {0x2fff, 2}, {0x1000, 0}, + {0x1000, static_cast(UINT32_MAX) + 1}}) { + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, &segment)); + EXPECT_EQ(ERDMAMEM, errno); + } + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); + segment.urma_ctx = reinterpret_cast(1); + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, &segment)); + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = UINT64_MAX - 7; + segment.seg.len = 16; + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); +} + // --------------------------------------------------------------------------- // SupportedByUrma: only baidu_std. // --------------------------------------------------------------------------- From 82806c03ee976d094026432a044ae4ec6aadbfb1 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:25:03 -0700 Subject: [PATCH 10/29] Add aligned copied URMA sends for access fault diagnosis --- docs/cn/urma.md | 8 ++ docs/en/urma.md | 10 +++ src/brpc/urma/urma_endpoint.cpp | 23 +++++- src/brpc/urma/urma_helper.cpp | 42 ++++++++++ src/brpc/urma/urma_helper.h | 9 +++ test/brpc_urma_copy_unittest.cpp | 134 +++++++++++++++++++++++++++++++ test/brpc_urma_unittest.cpp | 67 ++++++++++++++++ 7 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 test/brpc_urma_copy_unittest.cpp diff --git a/docs/cn/urma.md b/docs/cn/urma.md index c207e7d7a1..c744bffb32 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -157,6 +157,7 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs | `--urma_rq_size` | 128 | 本地 JFR 深度 [16, 4096] | | `--urma_cqe_poll_once` | 32 | 每次 `urma_poll_jfc` 的上限 | | `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 | +| `--urma_copy_send` | false | 将主机发送数据复制合并到 64 字节对齐的注册缓冲区 | | `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | `--urma_device` | "" | URMA 设备名(空=首个) | | `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) | @@ -218,3 +219,10 @@ SQ/接收 credit 耗尽保留 EAGAIN 背压,ACK 提交失败向连接错误处 模式切换作用于新连接;存量连接使用已协商的传输,冷却到期后重新准入 URMA。 `example/rdma_performance/recovery_client` 支持 `--transport=urma`,沿用 `--mode`、`--mode_file` 和 CSV 统计,可用于双机故障注入。 + +## 发送访问错误对照 + +设置 `--urma_copy_send=true` 后,每个数据 WR 使用一个新的、64 字节对齐的注册缓冲区, +该缓冲区由发送槽位持有至完成;消息和附件内容保持一致。此模式增加一次 CPU 拷贝。 +可在两端保持设备、队列、消息大小等参数一致,比较原发送路径与复制路径的结果。 +启动日志显示 `copy_send`,失败日志显示完成方向及发送槽位的地址、长度和注册段范围。 diff --git a/docs/en/urma.md b/docs/en/urma.md index 29939960b9..c02e3d6c0b 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -168,6 +168,7 @@ All flags use the `urma_` prefix (mirroring RDMA's `rdma_` prefix): | `--urma_rq_size` | 128 | Local JFR depth [16, 4096] | | `--urma_cqe_poll_once` | 32 | Max CQEs per `urma_poll_jfc` | | `--urma_recv_zerocopy` | true | Zero-copy receives above `--urma_zerocopy_min_size` | +| `--urma_copy_send` | false | Copy and coalesce host payloads into a 64-byte aligned registered send buffer | | `--urma_zerocopy_min_size` | 512 | Receives smaller than this are copied | | `--urma_device` | "" | URMA device name (empty = first) | | `--urma_max_sge` | 0 | Max SGEs per WR (0 = device max) | @@ -236,3 +237,12 @@ counts TCP admissions. Mode changes apply to new connections. Existing connectio keep their negotiated transport; URMA admission resumes after cooldown. `example/rdma_performance/recovery_client --transport=urma` supports `--mode`, `--mode_file` and CSV statistics for two-host fault injection. + +## Comparing send access failures + +With `--urma_copy_send=true`, each data WR uses a fresh, 64-byte aligned registered +buffer held by its send slot until completion. Message and attachment contents are +preserved. This mode adds a CPU copy. Compare it with the original send path while +keeping device, queue and payload settings the same on both peers. Startup logs +include `copy_send`; failure logs include direction, send-slot addresses, lengths +and registered segment ranges. diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 2545a69987..58ead7ffa1 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -64,6 +64,7 @@ DECLARE_int32(urma_sq_size); DECLARE_int32(urma_rq_size); DECLARE_int32(urma_cqe_poll_once); DECLARE_bool(urma_recv_zerocopy); +DECLARE_bool(urma_copy_send); DECLARE_int32(urma_zerocopy_min_size); DECLARE_int32(urma_prepared_jetty_cnt); DECLARE_bool(urma_poller_yield); @@ -686,6 +687,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { errno = ENOTCONN; return -1; } + const bool copy_send = FLAGS_urma_copy_send; int max_sge = GetUrmaMaxSge(); if (max_sge < 1) { max_sge = 1; @@ -716,6 +718,10 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { size_t max_len = _remote_recv_block_size > 0 ? _remote_recv_block_size : GetUrmaRecvBlockSize(); + if (copy_send) { + max_len = std::min(max_len, GetUrmaRecvBlockSize() - + (URMA_SEND_BUFFER_ALIGNMENT - 1)); + } while (sge_index < static_cast(max_sge) && this_len < max_len && current < ndata) { auto* data = reinterpret_cast(from[current]); @@ -723,8 +729,10 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ++current; continue; } - ssize_t n = data->cut_into_sglist(sglist, &sge_index, to, - max_sge, max_len - this_len); + ssize_t n = copy_send ? + static_cast(data->cutn(to, max_len - this_len)) : + data->cut_into_sglist(sglist, &sge_index, to, + max_sge, max_len - this_len); if (n < 0) { if (errno == ERDMAMEM) { ReportFailure(UrmaFailureReason::ACCESS_ERROR); @@ -733,9 +741,18 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { } this_len += n; } - if (sge_index == 0) { + if (this_len == 0) { break; } + if (copy_send) { + if (!CopyUrmaSendBuffer(to, &sglist[0])) { + LOG(ERROR) << "Fail to prepare aligned URMA send buffer: length=" + << this_len << " errno=" << errno; + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + return -1; + } + sge_index = 1; + } urma_sg_t sg{sglist, static_cast(sge_index)}; urma_jfs_wr_t wr{}; diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index f7c94d4827..b603a41649 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -76,6 +76,8 @@ DEFINE_int32(urma_cqe_poll_once, 32, "Max completion entries polled per urma_poll_jfc call"); DEFINE_bool(urma_recv_zerocopy, true, "Use zero-copy for receives larger than --urma_zerocopy_min_size"); +DEFINE_bool(urma_copy_send, false, + "Copy host send payloads into a 64-byte aligned registered buffer"); DEFINE_int32(urma_zerocopy_min_size, 512, "Receives smaller than this many bytes are copied (not zero-copy)"); @@ -439,6 +441,45 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, return true; } +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*)) { + const size_t length = buffer->size(); + const size_t capacity = GetUrmaRecvBlockSize(); + if (length == 0 || capacity < URMA_SEND_BUFFER_ALIGNMENT || + length > capacity - (URMA_SEND_BUFFER_ALIGNMENT - 1)) { + errno = EMSGSIZE; + return false; + } + butil::IOBuf copied; + void* data = nullptr; + size_t padding = 0; + { + butil::IOBufAsZeroCopyOutputStream stream( + &copied, capacity + sizeof(butil::IOBuf::Block)); + int available = 0; + if (!stream.Next(&data, &available)) { + errno = ENOMEM; + return false; + } + padding = (URMA_SEND_BUFFER_ALIGNMENT - + reinterpret_cast(data) % URMA_SEND_BUFFER_ALIGNMENT) % + URMA_SEND_BUFFER_ALIGNMENT; + if (padding + length > static_cast(available)) { + errno = ENOMEM; + return false; + } + data = static_cast(data) + padding; + buffer->copy_to(data, length); + stream.BackUp(available - static_cast(padding + length)); + } + copied.pop_front(padding); + if (!InitUrmaLocalSge(sge, reinterpret_cast(data), length, lookup(data))) { + return false; + } + buffer->swap(copied); + return true; +} + static void GlobalRelease() { g_urma_available.store(false, butil::memory_order_release); if (g_mem_alloc_orig) { @@ -656,6 +697,7 @@ static bool GlobalUrmaInitializeImpl() { LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device << " max_sge=" << g_max_sge + << " copy_send=" << FLAGS_urma_copy_send << " buffer_size=" << g_pool_buffer_size << " buffer_count=" << g_pool->buffer_count(); return true; diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index c7439812d7..25d8d87c70 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -26,6 +26,10 @@ #include "bthread/types.h" #include "butil/atomicops.h" +namespace butil { +class IOBuf; +} + #if BRPC_WITH_URMA #include "urma_api.h" @@ -87,6 +91,11 @@ uint8_t GetUrmaJettyPriority(); bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, urma_target_seg_t* segment); +constexpr size_t URMA_SEND_BUFFER_ALIGNMENT = 64; +// Copy host payloads into one aligned block and transfer its ownership to buffer. +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*) = GetPoolSegFor); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/test/brpc_urma_copy_unittest.cpp b/test/brpc_urma_copy_unittest.cpp new file mode 100644 index 0000000000..da403f608a --- /dev/null +++ b/test/brpc_urma_copy_unittest.cpp @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_bool(urma_copy_send); +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); +DECLARE_int32(urma_max_sge); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class CopiedEchoService : public test::EchoService { + void Echo(google::protobuf::RpcController* base, const test::EchoRequest* request, + test::EchoResponse* response, google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +class UrmaCopiedRpcTest : public ::testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (!_mock) { + return; + } + brpc::urma::FLAGS_urma_copy_send = true; + brpc::urma::FLAGS_urma_buffer_count = 2048; + brpc::urma::FLAGS_urma_prepared_jetty_cnt = 0; + brpc::urma::FLAGS_urma_max_sge = 1; + brpc::urma::GlobalUrmaInitializeOrDie(); + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "Copied URMA RPC tests use the link-time mock"; + } + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + ASSERT_NE(nullptr, brpc::urma::GetUrmaContext()); + brpc::urma::FLAGS_urma_client_handshake_version = GetParam(); + } + static bool _mock; +}; +bool UrmaCopiedRpcTest::_mock = false; + +TEST_P(UrmaCopiedRpcTest, copies_payload_and_attachment_across_wrs_and_credit_returns) { + CopiedEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "single"; + co.timeout_ms = 3000; + co.max_retry = 0; + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + // Exceed the send window to exercise receive-credit ACKs as well as data WRs. + for (int round = 0; round < 40; ++round) { + const size_t size = round == 0 ? 0 : 20000; + const std::string payload(size, 'v'); + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message(payload); + cntl.request_attachment().append(payload); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(payload, response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + } + EXPECT_EQ(before.negotiated_connections + 2, + brpc::urma::GetUrmaRecoveryStatus().negotiated_connections); + server.Stop(0); + server.Join(); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaCopiedRpcTest, ::testing::Values(2, 3)); + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index da77a94d48..fb14511cfc 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -312,6 +312,73 @@ TEST(UrmaHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); } +static urma_target_seg_t g_copy_test_segment; +static int g_copy_test_releases; + +static urma_target_seg_t* ResolveCopyTestSegment(void* address) { + g_copy_test_segment = urma_target_seg_t{}; + g_copy_test_segment.urma_ctx = urma::GetUrmaContext(); + g_copy_test_segment.seg.ubva.va = reinterpret_cast(address); + g_copy_test_segment.seg.len = urma::GetUrmaRecvBlockSize(); + return &g_copy_test_segment; +} + +static urma_target_seg_t* RejectCopyTestSegment(void*) { + return nullptr; +} + +static void DeleteCopyTestData(void* data) { + ++g_copy_test_releases; + delete[] static_cast(data); +} + +TEST(UrmaHelperTest, copied_send_coalesces_host_fragments_and_retains_aligned_payload) { + g_copy_test_releases = 0; + butil::IOBuf buffer; + for (const char* text : {"first", "second"}) { + const size_t length = strlen(text); + char* data = new char[length]; + memcpy(data, text, length); + ASSERT_EQ(0, buffer.append_user_data(data, length, DeleteCopyTestData)); + } + ASSERT_EQ(2u, buffer.backing_block_num()); + urma_sge_t sge{}; + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(2, g_copy_test_releases); + EXPECT_EQ("firstsecond", buffer.to_string()); + ASSERT_EQ(1u, buffer.backing_block_num()); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), sge.addr); + EXPECT_EQ(buffer.size(), sge.len); + EXPECT_EQ(&g_copy_test_segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + EXPECT_EQ("firstsecond", std::string(reinterpret_cast(sge.addr), sge.len)); + + buffer.clear(); + const std::string large(urma::GetUrmaRecvBlockSize() - + (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), 'x'); + buffer.append(large); + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(large, buffer.to_string()); + EXPECT_EQ(large.size(), sge.len); +} + +TEST(UrmaHelperTest, copied_send_failure_preserves_source_payload) { + butil::IOBuf buffer; + buffer.append("unchanged"); + urma_sge_t sge{}; + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); + EXPECT_EQ(ERDMAMEM, errno); + EXPECT_EQ("unchanged", buffer.to_string()); + const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); + buffer.clear(); + buffer.append(large); + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(EMSGSIZE, errno); + EXPECT_EQ(large, buffer.to_string()); +} + // --------------------------------------------------------------------------- // SupportedByUrma: only baidu_std. // --------------------------------------------------------------------------- From cd8e3e507e319b2e9b855edfe365c018a3e9435f Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:18:37 -0700 Subject: [PATCH 11/29] Align URMA QoS capability selection and hardware validation --- docs/cn/urma.md | 27 +++++++++++- docs/en/urma.md | 30 ++++++++++++- example/urma_performance/client.cpp | 5 ++- example/urma_performance/server.cpp | 3 +- src/brpc/urma/urma_endpoint.cpp | 16 ++++++- src/brpc/urma/urma_helper.cpp | 5 ++- src/brpc/urma/urma_helper.h | 4 +- test/brpc_urma_qos_unittest.cpp | 66 +++++++++++++++++++++++++++++ 8 files changed, 147 insertions(+), 9 deletions(-) diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 14105a02d1..827191e700 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -220,4 +220,29 @@ priority 对应的 SL 和调度待遇由设备及网络配置决定。 请求和响应分别配置各自的发送策略,v2/v3 握手格式沿用现有协议。 `example/urma_performance` 的 client/server 支持 `--urma_priority=10`。 -以不同 priority 启动竞争流量,对比吞吐和 P99,并结合设备 SL 映射验证调度效果。 +示例中的 6 和 10 须在所选设备上报告 CTP 能力;数字大小不代表调度优先关系。 + +### 实机验证 + +先使用同一对设备验证原生 RM、CTP、共享 JFR 数据通路。设置本机 `URMA_DEVICE`, +客户端另设 `SERVER_IP`,先启动服务端,再启动客户端: + +```bash +# 服务端 +urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ + -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 +# 客户端 +urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ + -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -S "$SERVER_IP" +``` + +两端再将 `-I 0` 改为 `-I 64` 做 inline 对照。原生测试通过后,用 +`--urma_priority=-1` 完成 brpc 单流基线,再逐个测试设备支持的显式 priority, +请求、响应方向分别改变配置。连接调试信息应为 `state=ESTABLISHED`,并显示预期的 +`urma_effective_priority`;TCP 回落的吞吐数据不计入 URMA QoS 结果。 +最后启动竞争流量,结合设备 SL 映射、队列计数器、吞吐和 P99 验证调度效果。 + +记录两端内核、UMDK、provider 和固件版本。`status=4` 是本地内存访问错误; +原生测试出现该错误时先定位数据通路。brpc 完成错误日志包含收发方向、配置 priority +和实际 Jetty priority,初始化日志包含 `default_priority`。mock 测试验证配置传递和 +Jetty 属性,报文标记与调度效果由实机测试验证。 diff --git a/docs/en/urma.md b/docs/en/urma.md index caa5eb09b3..bbd1613e16 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -237,4 +237,32 @@ the configured and effective priority. Each peer configures its outgoing directi v2/v3 handshake formats remain the same. The client and server in `example/urma_performance` accept `--urma_priority=10`. -Compare throughput and P99 under competing priorities using the device SL mapping. +The example values 6 and 10 require advertised CTP support on the selected device; +their numeric order does not specify scheduling precedence. + +### Hardware validation + +First test the same device pair with native RM, CTP and shared JFR. Set the local +`URMA_DEVICE` on each host and `SERVER_IP` on the client. Start the server first: + +```bash +# Server +urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ + -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 +# Client +urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ + -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -S "$SERVER_IP" +``` + +Repeat with `-I 64` on both hosts for the inline comparison. Once native tests +pass, establish a single-flow brpc baseline with `--urma_priority=-1`, then test +supported explicit priorities, changing the request and response directions +separately. Connection debug output must show `state=ESTABLISHED` and the expected +`urma_effective_priority`. Exclude TCP fallback measurements from URMA QoS results. +Then compare device SL mappings, queue counters, throughput and P99 under contention. + +Record kernel, UMDK, provider and firmware versions on both hosts. Status 4 means +a local memory access error; resolve native data-path failures before measuring +QoS. brpc completion errors log the direction and configured/effective priorities; +initialization logs `default_priority`. Mock tests check configuration propagation +and Jetty attributes; hardware tests check packet markings and scheduling effects. diff --git a/example/urma_performance/client.cpp b/example/urma_performance/client.cpp index a66043b2d7..2487e42f0a 100644 --- a/example/urma_performance/client.cpp +++ b/example/urma_performance/client.cpp @@ -112,7 +112,9 @@ int main(int argc, char* argv[]) { stub.Test(&warmup_cntl, &warmup_req, &warmup_resp, nullptr); if (warmup_cntl.Failed()) { LOG(ERROR) << "Warm-up RPC failed after timeout_ms=" - << FLAGS_rpc_timeout_ms << ": " + << FLAGS_rpc_timeout_ms + << " use_urma=" << FLAGS_use_urma + << " urma_priority=" << FLAGS_urma_priority << ": " << warmup_cntl.ErrorText(); return -1; } @@ -133,6 +135,7 @@ int main(int argc, char* argv[]) { } LOG(INFO) << "URMA performance client started (server=" << FLAGS_server << ", use_urma=" << FLAGS_use_urma + << ", urma_priority=" << FLAGS_urma_priority << ", threads=" << thread_num << ", rpc_timeout_ms=" << FLAGS_rpc_timeout_ms << ")"; while (!brpc::IsAskedToQuit()) { diff --git a/example/urma_performance/server.cpp b/example/urma_performance/server.cpp index b874b6a424..d3eddc1993 100644 --- a/example/urma_performance/server.cpp +++ b/example/urma_performance/server.cpp @@ -82,7 +82,8 @@ int main(int argc, char* argv[]) { return -1; } LOG(INFO) << "URMA performance server started on port " << FLAGS_port - << " (use_urma=" << FLAGS_use_urma << ")"; + << " (use_urma=" << FLAGS_use_urma + << ", urma_priority=" << FLAGS_urma_priority << ")"; server.RunUntilAskedToQuit(); return 0; } diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index c3fc26e1af..35c1821f26 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -932,7 +932,21 @@ int UrmaEndpoint::SendAck(int num) { ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { - LOG(WARNING) << "URMA completion failed, status=" << cr.status; + LOG(WARNING) << "URMA completion failed, status=" << cr.status + << " direction=" << (cr.flag.bs.s_r ? "recv" : "send") + << " flags=" << static_cast(cr.flag.value) + << " opcode=" << cr.opcode + << " local_id=" << cr.local_id + << " tpn=" << cr.tpn + << " user_ctx=" << cr.user_ctx + << " completion_len=" << cr.completion_len + << " urma_priority=" << _socket->urma_qos().priority + << " urma_effective_priority=" + << (_resource && _resource->jetty ? + static_cast( + _resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) + << " state=" << GetStateStr() + << " on " << _socket->description(); errno = EIO; return -1; } diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index e15bef6494..6a39630860 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -628,6 +628,7 @@ static bool GlobalUrmaInitializeImpl() { // resources on exit; GlobalRelease remains available for init rollback. LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device + << " default_priority=" << static_cast(g_jetty_priority) << " max_sge=" << g_max_sge << " buffer_size=" << g_pool_buffer_size << " buffer_count=" << g_pool->buffer_count(); @@ -679,8 +680,8 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type) { const union urma_tp_type_en expected = TpTypeCapability(tp_type); for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { - if (attr.dev_cap.priority_info[priority].tp_type.value == - expected.value) { + if ((attr.dev_cap.priority_info[priority].tp_type.value & + expected.value) != 0) { return priority; } } diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index de663e9906..c2bd2bcc05 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -76,8 +76,8 @@ const urma_eid_t* GetUrmaLocalEid(); // Return true when the selected URMA device is a bonding provider device. bool IsUrmaBondingDevice(); -// Find the priority whose advertised transport-path capability exactly -// matches @tp_type. Returns -1 when the device does not report one. +// Find the first priority whose advertised capabilities include @tp_type. +// Returns -1 when the device does not report one. int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type); diff --git a/test/brpc_urma_qos_unittest.cpp b/test/brpc_urma_qos_unittest.cpp index 7e63bb5878..9709cde944 100644 --- a/test/brpc_urma_qos_unittest.cpp +++ b/test/brpc_urma_qos_unittest.cpp @@ -22,6 +22,7 @@ #include #if BRPC_WITH_URMA +#include "butil/logging.h" #include "brpc/acceptor.h" #include "brpc/channel.h" #include "brpc/controller.h" @@ -48,6 +49,9 @@ class UrmaQosTestPeer { static int Priority(UrmaEndpoint* ep) { return ep->_resource->jetty->jetty_cfg.jfs_cfg.priority; } + static ssize_t Complete(UrmaEndpoint* ep, const urma_cr_t& cr) { + return ep->HandleCompletion(cr); + } }; } // namespace urma @@ -66,6 +70,24 @@ TEST(UrmaHelperTest, explicit_priority_requires_ctp_capability) { } } +TEST(UrmaHelperTest, automatic_priority_accepts_combined_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + EXPECT_EQ(3, urma::FindUrmaPriorityForTpType(attr, URMA_RTP)); + EXPECT_EQ(10, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 10)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( + attr, static_cast(-1))); + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 0; + EXPECT_EQ(15, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 0; + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); +} + class UrmaQosTest : public ::testing::Test { protected: static void SetUpTestSuite() { @@ -196,6 +218,50 @@ TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { } } +TEST_F(UrmaQosTest, access_error_preserves_priority_and_reports_direction) { + for (int priority : {-1, 10}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.priority = priority; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = Endpoint(socket); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); + const int effective = + priority == -1 ? urma::GetUrmaJettyPriority() : priority; + for (bool receive : {false, true}) { + urma_cr_t cr{}; + cr.status = URMA_CR_LOC_ACCESS_ERR; + cr.flag.bs.s_r = receive; + cr.local_id = 42; +#if !BRPC_WITH_GLOG + logging::StringSink log; + logging::LogSink* old_sink = logging::SetLogSink(&log); +#endif + const ssize_t result = urma::UrmaQosTestPeer::Complete(ep, cr); + const int saved_errno = errno; +#if !BRPC_WITH_GLOG + logging::SetLogSink(old_sink); + EXPECT_NE(std::string::npos, log.find("status=4")) << log; + EXPECT_NE(std::string::npos, + log.find(receive ? "direction=recv" : "direction=send")) << log; + EXPECT_NE(std::string::npos, log.find("local_id=42")) << log; + EXPECT_NE(std::string::npos, + log.find("urma_priority=" + std::to_string(priority))) << log; + EXPECT_NE(std::string::npos, + log.find("urma_effective_priority=" + + std::to_string(effective))) << log; +#endif + EXPECT_EQ(-1, result); + EXPECT_EQ(EIO, saved_errno); + EXPECT_EQ(effective, urma::UrmaQosTestPeer::Priority(ep)); + } + socket->SetFailed(); + } +} + class UrmaQosEcho : public ::test::EchoService { void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, ::test::EchoResponse* response, google::protobuf::Closure* done) override { From cc79e5337cd698fbbb9628a5058c479bc08e5e03 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Thu, 29 Oct 2026 20:30:04 -0700 Subject: [PATCH 12/29] Build recovery client from URMA performance example --- docs/cn/urma.md | 17 ++++++++++++++++- docs/en/urma.md | 19 ++++++++++++++++++- example/urma_performance/CMakeLists.txt | 5 +++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/cn/urma.md b/docs/cn/urma.md index c744bffb32..9aec1a84b7 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -218,7 +218,22 @@ SQ/接收 credit 耗尽保留 EAGAIN 背压,ACK 提交失败向连接错误处 `/vars/urma_recovery_tcp_fallback_admissions` 提供 TCP 准入次数。 模式切换作用于新连接;存量连接使用已协商的传输,冷却到期后重新准入 URMA。 -`example/rdma_performance/recovery_client` 支持 `--transport=urma`,沿用 `--mode`、`--mode_file` 和 CSV 统计,可用于双机故障注入。 +`example/urma_performance` 构建 `urma_recovery_client`,复用 RDMA 示例的恢复客户端源码, +通过 URMA 示例链接依赖,支持仅启用 URMA 的构建。增加 `--transport=urma` 选择 URMA, +使用 `--mode`、`--mode_file` 和 CSV 统计验证双机功能: + +```bash +cmake --build build/urma-example --target urma_performance_server urma_recovery_client -j8 +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE \ + --mode=auto --connection_type=short --seconds=15 --interval_ms=200 +``` + +`build/urma-example` 使用本文编译配置生成,链接当前 URMA brpc 静态库。 +先分别运行 `auto` 与 `tcp_only`,检查 RPC 成功数、URMA 协商数和 TCP 准入数。 +运行中通过 `--mode_file` 指向的文件切换 `auto`、`tcp_only`,使用 short 连接观察新准入。 +客户端退出码 0 表示至少一次 RPC 成功;健康基线还需确认 `failure=0`。 +功能测试检查模式切换、故障计数及恢复行为,吞吐和 P99 另作性能测试。 ## 发送访问错误对照 diff --git a/docs/en/urma.md b/docs/en/urma.md index c02e3d6c0b..4d91d10d13 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -236,7 +236,24 @@ cooldown state and negotiation counts. /vars/urma_recovery_tcp_fallback_admissio counts TCP admissions. Mode changes apply to new connections. Existing connections keep their negotiated transport; URMA admission resumes after cooldown. -`example/rdma_performance/recovery_client --transport=urma` supports `--mode`, `--mode_file` and CSV statistics for two-host fault injection. +`example/urma_performance` builds `urma_recovery_client` from the shared recovery +client source using the URMA example dependencies, including URMA-only builds. +Select URMA with `--transport=urma` and use modes and CSV counters for two-host checks: + +```bash +cmake --build build/urma-example --target urma_performance_server urma_recovery_client -j8 +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE \ + --mode=auto --connection_type=short --seconds=15 --interval_ms=200 +``` + +Configure `build/urma-example` as described above, linking the current URMA brpc +static library. Run `auto` and `tcp_only` separately and check RPC success, +URMA negotiation and TCP admission counters. Use `--mode_file` to switch between +`auto` and `tcp_only` while running; short connections exercise new admission. +Exit code 0 means at least one RPC succeeded; a healthy baseline also requires +`failure=0`. Functional checks cover mode changes, fault counters and recovery; +throughput and P99 belong to a separate performance test. ## Comparing send access failures diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 4dbc989a45..38b58dd6ff 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -52,3 +52,8 @@ brpc_example_configure_target(urma_performance_server) target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +add_executable(urma_recovery_client + ../rdma_performance/recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) +brpc_example_configure_target(urma_recovery_client) +target_link_libraries(urma_recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) From d97485c5550db26e7d9547df2902b5fc33dbccf8 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Fri, 30 Oct 2026 03:01:12 -0700 Subject: [PATCH 13/29] Resolve URMA business SL through device CTP priority mappings --- docs/cn/urma.md | 43 +++--- docs/en/urma.md | 55 +++++--- example/urma_performance/client.cpp | 6 +- example/urma_performance/server.cpp | 5 +- src/brpc/channel.cpp | 4 +- src/brpc/server.cpp | 3 +- src/brpc/urma/mock_urma.cpp | 3 + src/brpc/urma/urma_endpoint.cpp | 9 ++ src/brpc/urma/urma_helper.cpp | 77 +++++++++- src/brpc/urma/urma_helper.h | 9 +- src/brpc/urma/urma_qos.h | 10 +- test/brpc_urma_qos_unittest.cpp | 209 +++++++++++++++++++++++++++- 12 files changed, 373 insertions(+), 60 deletions(-) diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 827191e700..c08cd5b636 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -194,33 +194,39 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing - 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 - 当前实现面向 Linux。 -## 业务优先级 +## 业务 SL 与优先级 -`ChannelOptions.urma_qos.priority` 控制请求发送方向, -`ServerOptions.urma_qos.priority` 控制响应发送方向: +`ChannelOptions.urma_qos` 控制请求发送方向, +`ServerOptions.urma_qos` 控制响应发送方向: ```cpp brpc::ChannelOptions client_options; client_options.socket_mode = brpc::SOCKET_MODE_URMA; -client_options.urma_qos.priority = 10; +client_options.urma_qos.service_level = 6; brpc::ServerOptions server_options; server_options.socket_mode = brpc::SOCKET_MODE_URMA; -server_options.urma_qos.priority = 6; +server_options.urma_qos.service_level = 7; ``` -默认值 -1 沿用设备自动选择,取设备能力表中首个 CTP priority;未报告时沿用兼容值 6。 -显式值范围为 0–15,并要求所选设备的 `priority_info[priority].tp_type` 包含 CTP。 -不满足范围或能力要求时,Channel 初始化和 Server 启动返回错误。 -priority 对应的 SL 和调度待遇由设备及网络配置决定。 +当前使用 RM+CTP。CTP 的 SL 由设备 `priority_info[priority].SL` 决定, +业务 SL 与 priority 数字可以不同。`service_level` 默认为 -1,表示不指定 SL; +显式范围为 0–255。指定 SL 后,选择首个同时匹配该 SL 且包含 CTP 能力的 priority。 +`priority` 可单独指定,范围为 0–15;同时指定 SL 和 priority 时,两者必须匹配设备映射。 +找不到 CTP 映射或配置冲突时,Channel 初始化和 Server 启动报错,列出支持的 priority:SL。 +例如设备只有 priority 6→SL 6、priority 7→SL 7 支持 CTP 时,SL 10 会被拒绝。 -不同 priority 使用独立连接和 Jetty,相同配置共享连接。配置继承到服务发现、连接池、 +两项均为 -1 时沿用原自动选择:首个 CTP priority,未报告时使用兼容值 6。 +设备全局映射与调度待遇由平台配置;brpc 使用初始化时读取的映射,平台修改后重启进程加载。 +本接口按当前 RM+CTP 路径校验。 + +不同 SL/priority 配置使用独立连接,相同配置共享连接。配置继承到服务发现、连接池、 短连接和服务端 accept 路径,重建时重新应用。默认预建池只用于匹配其 priority 的连接, -其他 priority 按需创建 Jetty。连接调试信息显示配置值和实际 Jetty priority。 +其他 priority 按需创建 Jetty。连接调试信息显示请求 SL/priority、Jetty priority 和映射 SL。 请求和响应分别配置各自的发送策略,v2/v3 握手格式沿用现有协议。 -`example/urma_performance` 的 client/server 支持 `--urma_priority=10`。 -示例中的 6 和 10 须在所选设备上报告 CTP 能力;数字大小不代表调度优先关系。 +`example/urma_performance` 的 client/server 支持 `--urma_sl=6` 和 `--urma_priority=6`。 +示例值须在设备上有匹配的 CTP 映射,数字大小不代表调度优先关系。 ### 实机验证 @@ -237,12 +243,13 @@ urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ ``` 两端再将 `-I 0` 改为 `-I 64` 做 inline 对照。原生测试通过后,用 -`--urma_priority=-1` 完成 brpc 单流基线,再逐个测试设备支持的显式 priority, -请求、响应方向分别改变配置。连接调试信息应为 `state=ESTABLISHED`,并显示预期的 -`urma_effective_priority`;TCP 回落的吞吐数据不计入 URMA QoS 结果。 +默认 SL/priority 完成 brpc 单流基线,再逐个测试设备支持的显式 SL, +请求、响应方向分别改变配置,并验证不支持的 SL 和冲突配置在初始化时被拒绝。 +连接调试信息应为 `state=ESTABLISHED`,并显示预期的 `urma_effective_priority` 与 +`urma_effective_sl`;TCP 回落的吞吐数据不计入 URMA QoS 结果。 最后启动竞争流量,结合设备 SL 映射、队列计数器、吞吐和 P99 验证调度效果。 记录两端内核、UMDK、provider 和固件版本。`status=4` 是本地内存访问错误; -原生测试出现该错误时先定位数据通路。brpc 完成错误日志包含收发方向、配置 priority -和实际 Jetty priority,初始化日志包含 `default_priority`。mock 测试验证配置传递和 +原生测试出现该错误时先定位数据通路。brpc 完成错误日志包含收发方向、请求 SL/priority、 +Jetty priority 和映射 SL,初始化日志包含 `default_priority`、`default_sl`。mock 测试验证配置传递和 Jetty 属性,报文标记与调度效果由实机测试验证。 diff --git a/docs/en/urma.md b/docs/en/urma.md index bbd1613e16..5ed721b39b 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -208,37 +208,45 @@ is tracked as a future enhancement. - A hardware data path requires a supported UMDK provider and `liburma`. - The current implementation targets Linux. -## Business priority +## Business SL and priority -Set `ChannelOptions.urma_qos.priority` for outgoing requests and -`ServerOptions.urma_qos.priority` for outgoing responses: +Set `ChannelOptions.urma_qos` for outgoing requests and +`ServerOptions.urma_qos` for outgoing responses: ```cpp brpc::ChannelOptions client_options; client_options.socket_mode = brpc::SOCKET_MODE_URMA; -client_options.urma_qos.priority = 10; +client_options.urma_qos.service_level = 6; brpc::ServerOptions server_options; server_options.socket_mode = brpc::SOCKET_MODE_URMA; -server_options.urma_qos.priority = 6; +server_options.urma_qos.service_level = 7; ``` -The default -1 keeps automatic selection: the first CTP priority reported by the -device, with the existing compatibility value 6 when none is reported. Explicit -values are in [0,15] and require CTP in `priority_info[priority].tp_type` on the -selected device. Channel initialization and Server startup reject unsupported -values. Device and network configuration determine SL mapping and scheduling. - -Different priorities use separate connections and Jettys; equal configurations +The current path uses RM and CTP. CTP derives SL from +`priority_info[priority].SL`; SL and priority need not have equal numeric values. +`service_level` defaults to -1 (unspecified), with explicit values in [0,255]. +An explicit SL selects the first priority with both that SL and CTP capability. +`priority` can also be set explicitly in [0,15]. When both are specified, the +device mapping must satisfy both. Channel initialization and Server startup +reject unsupported or conflicting requests and report supported priority:SL pairs. +For example, if only priority 6/SL 6 and priority 7/SL 7 support CTP, SL 10 is rejected. + +With both fields at -1, automatic selection retains the first CTP priority and +the existing compatibility value 6 when none is reported. The platform configures +device mappings and scheduling; brpc uses the initialization snapshot. Restart +processes after changing platform mappings. Validation applies to the RM/CTP path. + +Different SL/priority configurations use separate connections; equal configurations share connections. Naming services, pooled and short connections, accepted sockets and rebuilt Jettys inherit the configuration. The prepared pool serves matching -priorities; other priorities allocate Jettys on demand. Debug output shows both -the configured and effective priority. Each peer configures its outgoing direction; +priorities; other priorities allocate Jettys on demand. Debug output shows the +requested SL/priority, Jetty priority and mapped SL. Each peer configures its outgoing direction; v2/v3 handshake formats remain the same. -The client and server in `example/urma_performance` accept `--urma_priority=10`. -The example values 6 and 10 require advertised CTP support on the selected device; -their numeric order does not specify scheduling precedence. +The client and server in `example/urma_performance` accept `--urma_sl=6` and +`--urma_priority=6`. Example values require matching CTP mappings on the selected +device; their numeric order does not specify scheduling precedence. ### Hardware validation @@ -255,14 +263,15 @@ urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ ``` Repeat with `-I 64` on both hosts for the inline comparison. Once native tests -pass, establish a single-flow brpc baseline with `--urma_priority=-1`, then test -supported explicit priorities, changing the request and response directions -separately. Connection debug output must show `state=ESTABLISHED` and the expected -`urma_effective_priority`. Exclude TCP fallback measurements from URMA QoS results. +pass, establish a single-flow brpc baseline with default SL/priority, then test +supported explicit SLs, changing the request and response directions separately. +Verify that unsupported SLs and conflicting settings fail initialization. +Connection debug output must show `state=ESTABLISHED` and the expected +`urma_effective_priority` and `urma_effective_sl`. Exclude TCP fallback measurements from URMA QoS results. Then compare device SL mappings, queue counters, throughput and P99 under contention. Record kernel, UMDK, provider and firmware versions on both hosts. Status 4 means a local memory access error; resolve native data-path failures before measuring -QoS. brpc completion errors log the direction and configured/effective priorities; -initialization logs `default_priority`. Mock tests check configuration propagation +QoS. brpc completion errors log the direction, requested SL/priority, Jetty priority +and mapped SL; initialization logs `default_priority` and `default_sl`. Mock tests check configuration propagation and Jetty attributes; hardware tests check packet markings and scheduling effects. diff --git a/example/urma_performance/client.cpp b/example/urma_performance/client.cpp index 2487e42f0a..37aa844bec 100644 --- a/example/urma_performance/client.cpp +++ b/example/urma_performance/client.cpp @@ -45,6 +45,7 @@ DEFINE_int32(rpc_timeout_ms, 5000, "Timeout for each RPC in milliseconds"); DEFINE_bool(echo_attachment, false, "Select whether attachment should be echo"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); DEFINE_int32(urma_priority, -1, "Outbound URMA priority: -1 automatic, or [0,15]"); +DEFINE_int32(urma_sl, -1, "Outbound URMA SL: -1 unspecified, or [0,255]"); bvar::LatencyRecorder g_latency("client"); bvar::Adder g_error_count("client_error_count"); @@ -90,6 +91,7 @@ int main(int argc, char* argv[]) { gflags::ParseCommandLineFlags(&argc, &argv, true); brpc::ChannelOptions options; options.urma_qos.priority = FLAGS_urma_priority; + options.urma_qos.service_level = FLAGS_urma_sl; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; options.connect_timeout_ms = FLAGS_rpc_timeout_ms; @@ -114,7 +116,8 @@ int main(int argc, char* argv[]) { LOG(ERROR) << "Warm-up RPC failed after timeout_ms=" << FLAGS_rpc_timeout_ms << " use_urma=" << FLAGS_use_urma - << " urma_priority=" << FLAGS_urma_priority << ": " + << " urma_priority=" << FLAGS_urma_priority + << " urma_sl=" << FLAGS_urma_sl << ": " << warmup_cntl.ErrorText(); return -1; } @@ -136,6 +139,7 @@ int main(int argc, char* argv[]) { LOG(INFO) << "URMA performance client started (server=" << FLAGS_server << ", use_urma=" << FLAGS_use_urma << ", urma_priority=" << FLAGS_urma_priority + << ", urma_sl=" << FLAGS_urma_sl << ", threads=" << thread_num << ", rpc_timeout_ms=" << FLAGS_rpc_timeout_ms << ")"; while (!brpc::IsAskedToQuit()) { diff --git a/example/urma_performance/server.cpp b/example/urma_performance/server.cpp index d3eddc1993..58c0d2ccaf 100644 --- a/example/urma_performance/server.cpp +++ b/example/urma_performance/server.cpp @@ -31,6 +31,7 @@ DEFINE_int32(port, 8003, "TCP Port of this server"); DEFINE_bool(use_urma, true, "Use URMA transport (true) or TCP (false)"); DEFINE_int32(urma_priority, -1, "Outbound URMA priority: -1 automatic, or [0,15]"); +DEFINE_int32(urma_sl, -1, "Outbound URMA SL: -1 unspecified, or [0,255]"); butil::atomic g_last_time(0); @@ -75,6 +76,7 @@ int main(int argc, char* argv[]) { brpc::ServerOptions options; options.urma_qos.priority = FLAGS_urma_priority; + options.urma_qos.service_level = FLAGS_urma_sl; options.socket_mode = FLAGS_use_urma ? brpc::SOCKET_MODE_URMA : brpc::SOCKET_MODE_TCP; if (server.Start(FLAGS_port, &options) != 0) { @@ -83,7 +85,8 @@ int main(int argc, char* argv[]) { } LOG(INFO) << "URMA performance server started on port " << FLAGS_port << " (use_urma=" << FLAGS_use_urma - << ", urma_priority=" << FLAGS_urma_priority << ")"; + << ", urma_priority=" << FLAGS_urma_priority + << ", urma_sl=" << FLAGS_urma_sl << ")"; server.RunUntilAskedToQuit(); return 0; } diff --git a/src/brpc/channel.cpp b/src/brpc/channel.cpp index ea87d58e7e..3c2ee44680 100644 --- a/src/brpc/channel.cpp +++ b/src/brpc/channel.cpp @@ -145,6 +145,7 @@ static ChannelSignature ComputeChannelSignature(const ChannelOptions& opt) { } else if (opt.socket_mode == SOCKET_MODE_URMA) { buf.append("|urma"); buf.append("|priority=").append(std::to_string(opt.urma_qos.priority)); + buf.append("|sl=").append(std::to_string(opt.urma_qos.service_level)); } butil::MurmurHash3_x64_128_Update(&mm_ctx, buf.data(), buf.size()); buf.clear(); @@ -196,7 +197,8 @@ int Channel::InitChannelOptions(const ChannelOptions* options) { return -1; } if (_options.socket_mode == SOCKET_MODE_URMA && !_options.urma_qos.IsValid()) { - LOG(ERROR) << "URMA priority must be -1 (automatic) or in [0,15]"; + LOG(ERROR) << "URMA priority must be -1 or in [0,15], " + "and SL must be -1 or in [0,255]"; return -1; } const Protocol* protocol = FindProtocol(_options.protocol); diff --git a/src/brpc/server.cpp b/src/brpc/server.cpp index 4ed8bba4ae..0564148d3f 100644 --- a/src/brpc/server.cpp +++ b/src/brpc/server.cpp @@ -881,7 +881,8 @@ int Server::StartInternal(const butil::EndPoint& endpoint, } if (real_opt.socket_mode == SOCKET_MODE_URMA && !real_opt.urma_qos.IsValid()) { - LOG(ERROR) << "URMA priority must be -1 (automatic) or in [0,15]"; + LOG(ERROR) << "URMA priority must be -1 or in [0,15], " + "and SL must be -1 or in [0,255]"; return -1; } diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index f81b54bf1a..853b8773f6 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -241,8 +241,11 @@ urma_status_t urma_query_device(urma_device_t *device, return URMA_EINVAL; } mock_device_attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[3].SL = 3; mock_device_attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[6].SL = 2; mock_device_attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[10].SL = 4; mock_device_attr.dev_cap.max_jfc = 1024; mock_device_attr.dev_cap.max_jetty = 1024; mock_device_attr.dev_cap.max_jfs_sge = 8; diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 35c1821f26..52864970ef 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -941,10 +941,15 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { << " user_ctx=" << cr.user_ctx << " completion_len=" << cr.completion_len << " urma_priority=" << _socket->urma_qos().priority + << " urma_sl=" << _socket->urma_qos().service_level << " urma_effective_priority=" << (_resource && _resource->jetty ? static_cast( _resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) + << " urma_effective_sl=" + << (_resource && _resource->jetty ? + GetUrmaServiceLevelForPriority( + _resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) << " state=" << GetStateStr() << " on " << _socket->description(); errno = EIO; @@ -1580,9 +1585,13 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) << " urma_priority=" << _socket->urma_qos().priority + << " urma_sl=" << _socket->urma_qos().service_level << " urma_effective_priority=" << (_resource && _resource->jetty ? static_cast(_resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) + << " urma_effective_sl=" + << (_resource && _resource->jetty ? GetUrmaServiceLevelForPriority( + _resource->jetty->jetty_cfg.jfs_cfg.priority) : -1) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index 6a39630860..3995089f4b 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -629,6 +630,7 @@ static bool GlobalUrmaInitializeImpl() { LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device << " default_priority=" << static_cast(g_jetty_priority) + << " default_sl=" << GetUrmaServiceLevelForPriority(g_jetty_priority) << " max_sge=" << g_max_sge << " buffer_size=" << g_pool_buffer_size << " buffer_count=" << g_pool->buffer_count(); @@ -694,24 +696,85 @@ bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority) { attr.dev_cap.priority_info[priority].tp_type.bs.ctp; } -bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { +static std::string DescribeUrmaCtpSlMap(const urma_device_attr_t& attr) { + std::ostringstream os; + os << '['; + bool first = true; + for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { + if (IsUrmaCtpPrioritySupported(attr, priority)) { + if (!first) { + os << ','; + } + first = false; + os << priority << ':' << attr.dev_cap.priority_info[priority].SL; + } + } + os << ']'; + return os.str(); +} + +bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, + const UrmaQosOptions& qos, uint8_t default_priority, + uint8_t* priority) { if (!qos.IsValid()) { errno = EINVAL; return false; } + if (qos.priority == -1 && qos.service_level == -1) { + *priority = default_priority; + return true; + } if (qos.priority == -1) { + for (int p = 0; p <= URMA_MAX_PRIORITY; ++p) { + if (IsUrmaCtpPrioritySupported(attr, p) && + attr.dev_cap.priority_info[p].SL == + static_cast(qos.service_level)) { + *priority = static_cast(p); + return true; + } + } + LOG(ERROR) << "URMA SL " << qos.service_level + << " has no CTP priority; supported priority:SL=" + << DescribeUrmaCtpSlMap(attr); + } else if (!IsUrmaCtpPrioritySupported(attr, qos.priority)) { + LOG(ERROR) << "URMA priority " << qos.priority + << " does not advertise CTP on the selected device; " + "requested SL=" << qos.service_level + << "; supported priority:SL=" << DescribeUrmaCtpSlMap(attr); + } else if (qos.service_level != -1 && + attr.dev_cap.priority_info[qos.priority].SL != + static_cast(qos.service_level)) { + LOG(ERROR) << "URMA priority " << qos.priority << " maps to SL " + << attr.dev_cap.priority_info[qos.priority].SL + << ", requested SL=" << qos.service_level + << "; supported priority:SL=" << DescribeUrmaCtpSlMap(attr); + } else { + *priority = static_cast(qos.priority); + return true; + } + errno = EINVAL; + return false; +} + +bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { + if (qos.IsValid() && qos.priority == -1 && qos.service_level == -1) { *priority = GetUrmaJettyPriority(); return true; } - if (!IsUrmaAvailable() || !g_context || - !IsUrmaCtpPrioritySupported(g_device_attr, qos.priority)) { - LOG(ERROR) << "URMA priority " << qos.priority - << " does not advertise CTP on the selected device"; + if (!IsUrmaAvailable() || !g_context) { + LOG(ERROR) << "URMA QoS requires an initialized device: priority=" + << qos.priority << " SL=" << qos.service_level; errno = EINVAL; return false; } - *priority = static_cast(qos.priority); - return true; + return ResolveUrmaJettyPriority(g_device_attr, qos, GetUrmaJettyPriority(), priority); +} + +int64_t GetUrmaServiceLevelForPriority(int priority) { + if (!g_context || !IsUrmaCtpPrioritySupported(g_device_attr, priority)) { + return -1; + } + return g_device_attr.dev_cap.priority_info[priority].SL; } int GetUrmaMaxSge() { return g_max_sge; } size_t GetUrmaRecvBlockSize() { return g_recv_block_size; } diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index c2bd2bcc05..274f5a7a0c 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -81,13 +81,20 @@ bool IsUrmaBondingDevice(); int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type); -// Return the priority selected for the CTP jettys created by brpc. +// Return the default CTP Jetty priority selected for the device. uint8_t GetUrmaJettyPriority(); // Check whether an explicit priority advertises the CTP path used by brpc. bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority); +// Resolve a requested priority/SL pair against a device capability snapshot. +// Fully automatic options retain default_priority, including legacy fallback. +bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, + const UrmaQosOptions& qos, uint8_t default_priority, + uint8_t* priority); // Resolve automatic selection or validate against the initialized device. bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority); +// Return the advertised CTP SL, or -1 when no initialized mapping is available. +int64_t GetUrmaServiceLevelForPriority(int priority); // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/src/brpc/urma/urma_qos.h b/src/brpc/urma/urma_qos.h index 5c8924bdee..6c0a1936d2 100644 --- a/src/brpc/urma/urma_qos.h +++ b/src/brpc/urma/urma_qos.h @@ -20,11 +20,15 @@ namespace brpc { -// Outbound Jetty priority. Device CTP capabilities define valid selections. +// Outbound CTP QoS, resolved through the selected device's priority/SL map. struct UrmaQosOptions { - int priority = -1; // -1 selects the device default; explicit range [0, 15]. + int priority = -1; // -1 selects from SL or device default; explicit [0, 15]. + int service_level = -1; // -1 leaves SL unspecified; explicit range [0, 255]. - bool IsValid() const { return priority >= -1 && priority <= 15; } + bool IsValid() const { + return priority >= -1 && priority <= 15 && + service_level >= -1 && service_level <= 255; + } }; } // namespace brpc diff --git a/test/brpc_urma_qos_unittest.cpp b/test/brpc_urma_qos_unittest.cpp index 9709cde944..bb04a9bee3 100644 --- a/test/brpc_urma_qos_unittest.cpp +++ b/test/brpc_urma_qos_unittest.cpp @@ -88,6 +88,73 @@ TEST(UrmaHelperTest, automatic_priority_accepts_combined_capabilities) { EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); } +TEST(UrmaHelperTest, service_level_selects_ctp_priority_from_mapping) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].SL = 6; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 7; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].SL = 6; + attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[15].SL = 6; + UrmaQosOptions qos; + uint8_t priority = 0; + qos.service_level = 6; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(10, priority); + qos.service_level = 7; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(6, priority); + qos.priority = 6; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(6, priority); + attr.dev_cap.priority_info[15].SL = 255; + qos.priority = -1; + qos.service_level = 255; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(15, priority); + qos.service_level = -1; + attr = urma_device_attr_t{}; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(6, priority); +} + +TEST(UrmaHelperTest, service_level_rejects_unsupported_and_conflicting_mapping) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[3].SL = 3; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 2; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].SL = 4; + UrmaQosOptions qos; + uint8_t priority = 255; + for (int sl : {-2, 0, 3, 6, 255, 256}) { + qos.service_level = sl; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ(255, priority); + } + qos.priority = 6; + qos.service_level = 4; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ(255, priority); +#if !BRPC_WITH_GLOG + qos.priority = -1; + qos.service_level = 3; + logging::StringSink log; + logging::LogSink* old_sink = logging::SetLogSink(&log); + const bool resolved = urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority); + logging::SetLogSink(old_sink); + EXPECT_FALSE(resolved); + EXPECT_NE(std::string::npos, log.find("SL 3 has no CTP priority")) << log; + EXPECT_NE(std::string::npos, log.find("[6:2,10:4]")) << log; +#endif +} + class UrmaQosTest : public ::testing::Test { protected: static void SetUpTestSuite() { @@ -162,6 +229,39 @@ TEST_F(UrmaQosTest, rejects_invalid_and_unsupported_priority) { } } +TEST_F(UrmaQosTest, rejects_invalid_unsupported_and_conflicting_service_level) { + struct RequestedQos { int priority; int sl; }; + for (const auto& requested : {RequestedQos{-1, -2}, {-1, 256}, {-1, 0}, + {-1, 3}, {-1, 6}, {-1, 255}, {6, 4}, {3, 3}}) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.priority = requested.priority; + co.urma_qos.service_level = requested.sl; + Channel channel; + EXPECT_EQ(-1, channel.Init("127.0.0.1:8123", &co)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos = co.urma_qos; + Server server; + EXPECT_EQ(-1, server.Start("127.0.0.1:0", &so)); + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos = co.urma_qos; + SocketId sid; + if (!co.urma_qos.IsValid()) { + EXPECT_EQ(-1, Socket::Create(options, &sid)); + EXPECT_EQ(EINVAL, errno); + } else { + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(EINVAL, errno); + socket->SetFailed(); + } + } +} + TEST_F(UrmaQosTest, channels_isolate_priority_and_inherit_to_child_connections) { ChannelOptions co; co.socket_mode = SOCKET_MODE_URMA; @@ -196,24 +296,68 @@ TEST_F(UrmaQosTest, channels_isolate_priority_and_inherit_to_child_connections) short_socket->SetFailed(); } +TEST_F(UrmaQosTest, service_level_isolates_and_inherits_connections) { + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.service_level = 2; + Channel low, same, high, named, by_priority, tcp, tcp_with_qos; + ASSERT_EQ(0, low.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, same.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, named.Init("list://127.0.0.1:8123", "rr", &co)); + co.urma_qos.service_level = 4; + ASSERT_EQ(0, high.Init("127.0.0.1:8123", &co)); + co.urma_qos.service_level = -1; + co.urma_qos.priority = 6; + ASSERT_EQ(0, by_priority.Init("127.0.0.1:8123", &co)); + EXPECT_EQ(low._server_id, same._server_id); + EXPECT_NE(low._server_id, high._server_id); + EXPECT_NE(low._server_id, by_priority._server_id); + SocketUniquePtr main, pooled, short_socket, selected; + ASSERT_EQ(0, Socket::Address(low._server_id, &main)); + ASSERT_EQ(0, main->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (auto* socket : {main.get(), pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(-1, socket->urma_qos().priority); + EXPECT_EQ(2, socket->urma_qos().service_level); + } + EXPECT_EQ(main->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); + co.socket_mode = SOCKET_MODE_TCP; + co.urma_qos.service_level = 256; + ASSERT_EQ(0, tcp_with_qos.Init("127.0.0.1:8123", &co)); + ASSERT_EQ(0, tcp.Init("127.0.0.1:8123", nullptr)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); +} + TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { urma::UrmaEndpoint::GlobalRelease(); ASSERT_EQ(0, urma::UrmaEndpoint::GlobalInitialize()); - // Allocate priority 10 before consuming the two prepared default Jettys. - for (int priority : {10, -1, 6, 10}) { + // SL 4 resolves to priority 10 and must leave the default Jettys available. + struct RequestedQos { int priority; int sl; int effective; }; + for (const auto& qos : {RequestedQos{-1, 4, 10}, {-1, 2, 6}, {6, 2, 6}, + {10, -1, 10}, {-1, -1, 6}, {10, 4, 10}}) { SocketOptions options; options.socket_mode = SOCKET_MODE_URMA; - options.urma_qos.priority = priority; + options.urma_qos.priority = qos.priority; + options.urma_qos.service_level = qos.sl; SocketId sid; ASSERT_EQ(0, Socket::Create(options, &sid)); SocketUniquePtr socket; ASSERT_EQ(0, Socket::Address(sid, &socket)); auto* ep = Endpoint(socket); ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); - EXPECT_EQ(priority == -1 ? 6 : priority, urma::UrmaQosTestPeer::Priority(ep)); + EXPECT_EQ(qos.effective, urma::UrmaQosTestPeer::Priority(ep)); std::ostringstream debug; ep->DebugInfo(debug); EXPECT_NE(std::string::npos, debug.str().find("urma_effective_priority=")); + EXPECT_NE(std::string::npos, debug.str().find( + "urma_sl=" + std::to_string(qos.sl))); + EXPECT_NE(std::string::npos, debug.str().find( + "urma_effective_sl=" + std::to_string(qos.effective == 6 ? 2 : 4))); socket->SetFailed(); } } @@ -223,6 +367,7 @@ TEST_F(UrmaQosTest, access_error_preserves_priority_and_reports_direction) { SocketOptions options; options.socket_mode = SOCKET_MODE_URMA; options.urma_qos.priority = priority; + options.urma_qos.service_level = priority == -1 ? 2 : 4; SocketId sid; ASSERT_EQ(0, Socket::Create(options, &sid)); SocketUniquePtr socket; @@ -253,6 +398,10 @@ TEST_F(UrmaQosTest, access_error_preserves_priority_and_reports_direction) { EXPECT_NE(std::string::npos, log.find("urma_effective_priority=" + std::to_string(effective))) << log; + EXPECT_NE(std::string::npos, log.find("urma_sl=" + + std::to_string(options.urma_qos.service_level))) << log; + EXPECT_NE(std::string::npos, log.find("urma_effective_sl=" + + std::to_string(options.urma_qos.service_level))) << log; #endif EXPECT_EQ(-1, result); EXPECT_EQ(EIO, saved_errno); @@ -313,6 +462,58 @@ TEST_F(UrmaQosTest, request_and_response_priorities_apply_across_restart) { } } +TEST_F(UrmaQosTest, request_and_response_service_levels_apply_across_restart) { + GFLAGS_NAMESPACE::FlagSaver saved_flags; + UrmaQosEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + for (int response_sl : {2, 4}) { + urma::FLAGS_urma_client_handshake_version = response_sl == 2 ? 2 : 3; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + so.urma_qos.service_level = response_sl; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.urma_qos.service_level = response_sl == 2 ? 4 : 2; + co.timeout_ms = 2000; + co.max_retry = 0; + co.connection_type = "single"; + Channel channel; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("URMA SL mapping"); + ::test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + SocketUniquePtr client; + ASSERT_EQ(0, Socket::Address(channel._server_id, &client)); + EXPECT_EQ(-1, client->urma_qos().priority); + EXPECT_EQ(co.urma_qos.service_level, client->urma_qos().service_level); + EXPECT_EQ(response_sl == 2 ? 10 : 6, + urma::UrmaQosTestPeer::Priority(Endpoint(client))); + std::vector sockets; + server._am->ListConnections(&sockets); + ASSERT_EQ(1u, sockets.size()); + SocketUniquePtr accepted; + ASSERT_EQ(0, Socket::Address(sockets[0], &accepted)); + EXPECT_EQ(-1, accepted->urma_qos().priority); + EXPECT_EQ(response_sl, accepted->urma_qos().service_level); + EXPECT_EQ(response_sl == 2 ? 6 : 10, + urma::UrmaQosTestPeer::Priority(Endpoint(accepted))); + std::ostringstream debug; + Endpoint(accepted)->DebugInfo(debug); + EXPECT_NE(std::string::npos, debug.str().find( + "urma_effective_sl=" + std::to_string(response_sl))); + accepted.reset(); + client.reset(); + server.Stop(0); + server.Join(); + } +} + #endif // BRPC_WITH_URMA int main(int argc, char** argv) { From 061c3996cf0e0c090b85df9f01b69306ce380678 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:13:49 -0700 Subject: [PATCH 14/29] Support RTP paths for URMA runtime recovery --- docs/cn/urma.md | 18 ++- docs/en/urma.md | 23 +++- src/brpc/urma/mock_urma.cpp | 4 + src/brpc/urma/urma_endpoint.cpp | 12 +- src/brpc/urma/urma_helper.cpp | 65 +++++++-- src/brpc/urma/urma_helper.h | 13 +- test/CMakeLists.txt | 4 + test/brpc_urma_transport_unittest.cpp | 186 ++++++++++++++++++++++++++ test/brpc_urma_unittest.cpp | 84 ++++++++++++ 9 files changed, 390 insertions(+), 19 deletions(-) create mode 100644 test/brpc_urma_transport_unittest.cpp diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 9aec1a84b7..edb19c2c96 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -11,7 +11,7 @@ UrmaTransport 是使用 openEuler ## 技术背景 URMA 在 UMDK 支持的设备上提供 verbs 风格接口。当前实现创建可靠消息 -(`URMA_TM_RM`)Jetty,并使用 CTP 传输路径;通过 +(`URMA_TM_RM`)Jetty,并使用 CTP 或 RTP 传输路径;通过 `urma_post_jetty_send_wr` 提交发送 WR,通过 `urma_post_jfr_wr` 提交接收 WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获取。 @@ -160,6 +160,7 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs | `--urma_copy_send` | false | 将主机发送数据复制合并到 64 字节对齐的注册缓冲区 | | `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | `--urma_device` | "" | URMA 设备名(空=首个) | +| `--urma_tp_type` | auto | RM 路径:auto 优先 CTP,其次 RTP;ctp 或 rtp 显式选择路径 | | `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) | | `--urma_bonding_mode` | 0 | bonding 模式:0=standalone,1=active-backup,2=balance | | `--urma_bonding_level` | 0 | bonding 层级:0=IODIE,1=port | @@ -195,6 +196,21 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing - 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 - 当前实现面向 Linux。 +### 传输路径选择 + +进程初始化时,`--urma_tp_type=auto` 从设备表中选择第一个支持 CTP 的优先级, +没有 CTP 时选择第一个支持 RTP 的优先级,同时支持组合能力位。 +v2/v3 握手携带所选路径,Jetty 使用对应优先级。两端需配置相同路径, +不匹配的对端路径会在导入资源前被拒绝。路径在进程生命周期内保持固定。 + +仅提供 RTP 的设备,两端均使用 `--urma_device=DEVICE --urma_tp_type=rtp`。 +优先级是设备表索引,映射的 SL 可以不同,例如 priority 0 映射 SL 2。 +brpc 按路径能力选择优先级,设备映射保持原值。初始化日志记录 `tp_type` 和 `priority`。 + +能力表已报告但不支持显式指定路径时,初始化沿用现有 TCP 回落流程。 +未报告任何路径能力的 provider 保留兼容值:auto/ctp 使用 CTP priority 6, +rtp 使用 RTP priority 0。验收 URMA 数据通路时检查协商计数。 + ## 运行期故障恢复 通过进程级 API 控制新连接的传输选择: diff --git a/docs/en/urma.md b/docs/en/urma.md index 4d91d10d13..d56b180ba7 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -10,7 +10,7 @@ for large-packet / cross-node scenarios (Route C, "double backend"). ## Technical Background URMA exposes a verbs-style API over devices supported by UMDK. This -implementation creates a reliable-message (`URMA_TM_RM`) Jetty with a CTP +implementation creates a reliable-message (`URMA_TM_RM`) Jetty with a CTP or RTP transport path, posts send work requests with `urma_post_jetty_send_wr`, and posts receive work requests with `urma_post_jfr_wr`. Completions are consumed from a JFC either by busy polling or through a JFCE event fd. @@ -171,6 +171,7 @@ All flags use the `urma_` prefix (mirroring RDMA's `rdma_` prefix): | `--urma_copy_send` | false | Copy and coalesce host payloads into a 64-byte aligned registered send buffer | | `--urma_zerocopy_min_size` | 512 | Receives smaller than this are copied | | `--urma_device` | "" | URMA device name (empty = first) | +| `--urma_tp_type` | auto | RM path: auto prefers CTP, then RTP; ctp or rtp selects a path explicitly | | `--urma_max_sge` | 0 | Max SGEs per WR (0 = device max) | | `--urma_bonding_mode` | 0 | Bonding mode: 0=standalone, 1=active-backup, 2=balance | | `--urma_bonding_level` | 0 | Bonding level: 0=IODIE, 1=port | @@ -209,6 +210,26 @@ is tracked as a future enhancement. - A hardware data path requires a supported UMDK provider and `liburma`. - The current implementation targets Linux. +### Transport path selection + +At process initialization, `--urma_tp_type=auto` selects the first priority +advertising CTP, or the first RTP priority when CTP is absent. Combined capability +bits are supported. Both handshake versions advertise the selected path, and +Jettys use its priority. Configure both peers for the same path; an incompatible +peer path is rejected before importing resources. The path remains fixed for +the process lifetime. + +For an RTP-only device, start both the server and client with +`--urma_device=DEVICE --urma_tp_type=rtp`. A priority value is a device table +index and can map to a different SL. For example, priority 0 may map to SL 2. +brpc selects the priority from the path capability table and leaves the mapping +unchanged. Initialization logs report `tp_type` and `priority`. + +An explicit path missing from a reported capability table disables URMA through +the existing initialization fallback. Providers reporting no path capabilities +retain compatibility values: CTP priority 6 for auto/ctp, RTP priority 0 for rtp. +Check negotiation counters when validating a URMA data path. + ## Runtime recovery Configure admission for new connections with the process-level API: diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index a1844d05e4..f6b89db1ec 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -244,6 +244,10 @@ urma_status_t urma_query_device(urma_device_t *device, mock_device_attr.dev_cap.max_jetty = 1024; mock_device_attr.dev_cap.max_jfs_sge = 8; mock_device_attr.dev_cap.max_jfr_sge = 8; + mock_device_attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + mock_device_attr.dev_cap.priority_info[0].SL = 2; + mock_device_attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[6].SL = 6; memcpy(attr, &mock_device_attr, sizeof(urma_device_attr_t)); return URMA_SUCCESS; } diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 58ead7ffa1..6cd1994f1d 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -265,7 +265,7 @@ void UrmaEndpoint::MakeLocalParsedHello(ParsedHello* out) const { : _resource->jetty->jetty_id.eid.raw; std::memcpy(out->eid, advertised_eid, 16); } - out->tp_type = static_cast(URMA_CTP); + out->tp_type = static_cast(GetUrmaTpType()); // Pool segment: flatten g_pool_seg's seg fields. urma_target_seg_t* pool = GetPoolSegFor(nullptr); if (pool) { @@ -542,6 +542,12 @@ void UrmaEndpoint::DeallocateResources() { // ============================================================================ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { + if (peer.tp_type != static_cast(GetUrmaTpType())) { + LOG(ERROR) << "URMA peer path mismatch: local=" << GetUrmaTpType() + << " peer=" << static_cast(peer.tp_type); + errno = EPROTO; + return -1; + } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { errno = ENODEV; @@ -575,10 +581,6 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { remote.jetty_id.id = peer.jetty_id; remote.trans_mode = URMA_TM_RM; remote.type = URMA_JETTY; - if (peer.tp_type > static_cast(URMA_UTP)) { - errno = EPROTO; - return -1; - } remote.tp_type = static_cast(peer.tp_type); urma_token_t token{}; diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index b603a41649..b9aa4b84c0 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -83,6 +83,8 @@ DEFINE_int32(urma_zerocopy_min_size, 512, DEFINE_string(urma_device, "", "The name of the URMA device to use. Empty means the first one."); +DEFINE_string(urma_tp_type, "auto", + "RM transport path: auto (prefer CTP, then RTP), ctp, or rtp"); DEFINE_int32(urma_max_sge, 0, "Max SGEs per WR. 0 means the device maximum."); DEFINE_int32(urma_bonding_mode, 0, @@ -126,6 +128,7 @@ static bool g_is_bonding_device = false; // Prefer the device capability table and retain priority 6 as a compatibility // fallback for CTP providers that do not report a priority. static uint8_t g_jetty_priority = 6; +static urma_tp_type_t g_tp_type = URMA_CTP; // urma_init/urma_uninit manage process-global liburma state. Only uninitialize // it when this helper performed the successful initialization; URMA_EEXIST // means another component owns that state. @@ -518,6 +521,7 @@ static void GlobalRelease() { g_has_local_eid = false; g_is_bonding_device = false; g_jetty_priority = 6; + g_tp_type = URMA_CTP; if (g_owns_urma_init) { const urma_status_t status = urma_uninit(); if (status != URMA_SUCCESS) { @@ -593,13 +597,22 @@ static bool GlobalUrmaInitializeImpl() { g_device = nullptr; return false; } - const int ctp_priority = - FindUrmaPriorityForTpType(g_device_attr, URMA_CTP); - if (ctp_priority >= 0) { - g_jetty_priority = static_cast(ctp_priority); - } else { - LOG(WARNING) << "URMA device does not report a CTP priority; " - "falling back to compatibility priority " + if (!SelectUrmaTransportPath(g_device_attr, FLAGS_urma_tp_type, + &g_tp_type, &g_jetty_priority)) { + LOG(ERROR) << "URMA path selection failed: requested=" + << FLAGS_urma_tp_type << " device=" << device_name + << " ctp_priority=" + << FindUrmaPriorityForTpType(g_device_attr, URMA_CTP) + << " rtp_priority=" + << FindUrmaPriorityForTpType(g_device_attr, URMA_RTP); + urma_free_device_list(devices); + g_device = nullptr; + errno = EINVAL; + return false; + } + if (FindUrmaPriorityForTpType(g_device_attr, g_tp_type) < 0) { + LOG(WARNING) << "URMA device does not report path capabilities; " + "using compatibility priority " << static_cast(g_jetty_priority); } uint32_t eid_cnt = 0; @@ -696,6 +709,8 @@ static bool GlobalUrmaInitializeImpl() { // resources on exit; GlobalRelease remains available for init rollback. LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device + << " tp_type=" << (g_tp_type == URMA_CTP ? "ctp" : "rtp") + << " priority=" << static_cast(g_jetty_priority) << " max_sge=" << g_max_sge << " copy_send=" << FLAGS_urma_copy_send << " buffer_size=" << g_pool_buffer_size @@ -747,14 +762,46 @@ bool IsUrmaBondingDevice() { return g_is_bonding_device; } int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type) { const union urma_tp_type_en expected = TpTypeCapability(tp_type); + if (expected.value == 0) { + return -1; + } for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { - if (attr.dev_cap.priority_info[priority].tp_type.value == - expected.value) { + if ((attr.dev_cap.priority_info[priority].tp_type.value & + expected.value) != 0) { return priority; } } return -1; } +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority) { + if (!tp_type || !priority || + (requested != "auto" && requested != "ctp" && requested != "rtp")) { + return false; + } + for (auto candidate : {URMA_CTP, URMA_RTP}) { + if ((requested == "ctp" && candidate != URMA_CTP) || + (requested == "rtp" && candidate != URMA_RTP)) { + continue; + } + const int match = FindUrmaPriorityForTpType(attr, candidate); + if (match >= 0) { + *tp_type = candidate; + *priority = static_cast(match); + return true; + } + } + for (int i = 0; i <= URMA_MAX_PRIORITY; ++i) { + if (attr.dev_cap.priority_info[i].tp_type.value != 0) { + return false; + } + } + *tp_type = requested == "rtp" ? URMA_RTP : URMA_CTP; + *priority = *tp_type == URMA_CTP ? 6 : 0; + return true; +} +urma_tp_type_t GetUrmaTpType() { return g_tp_type; } uint8_t GetUrmaJettyPriority() { return g_jetty_priority; } int GetUrmaMaxSge() { return g_max_sge; } size_t GetUrmaRecvBlockSize() { return g_recv_block_size; } diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 25d8d87c70..b46475f3da 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -79,12 +79,19 @@ const urma_eid_t* GetUrmaLocalEid(); // Return true when the selected URMA device is a bonding provider device. bool IsUrmaBondingDevice(); -// Find the priority whose advertised transport-path capability exactly -// matches @tp_type. Returns -1 when the device does not report one. +// Find the first priority advertising @tp_type, including combined capabilities. +// Returns -1 when the device does not report one. int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type); -// Return the priority selected for the CTP jettys created by brpc. +// Select an RM path from "auto", "ctp", or "rtp". Auto prefers CTP, then RTP. +// Providers without a capability table retain CTP/6 or explicit RTP/0. +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority); + +// Return the process-wide path and priority selected at initialization. +urma_tp_type_t GetUrmaTpType(); uint8_t GetUrmaJettyPriority(); // Initialize a local SGE whose complete byte range belongs to this context. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a0fc2d0536..2a32083158 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -295,6 +295,10 @@ foreach(BRPC_UT ${BRPC_UNITTESTS}) brpc_test_config ${GPERFTOOLS_LIBRARIES}) add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE}) + if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_transport_unittest") + add_test(NAME brpc_urma_transport_rtp_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + endif() endforeach() if(BUILD_FUZZ_TESTS) diff --git a/test/brpc_urma_transport_unittest.cpp b/test/brpc_urma_transport_unittest.cpp new file mode 100644 index 0000000000..cfd1dc84e4 --- /dev/null +++ b/test/brpc_urma_transport_unittest.cpp @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_handshake.h" +#include "brpc/urma/urma_handshake.pb.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_string(urma_tp_type); +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class PathEchoService : public test::EchoService { + void Echo(google::protobuf::RpcController* base, const test::EchoRequest* request, + test::EchoResponse* response, google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +class UrmaTransportTest : public ::testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (_mock) { + brpc::urma::FLAGS_urma_buffer_count = 4096; + brpc::urma::GlobalUrmaInitializeOrDie(); + } + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "Path RPC tests use the link-time URMA mock"; + } + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + const bool rtp = brpc::urma::FLAGS_urma_tp_type == "rtp"; + ASSERT_EQ(rtp ? URMA_RTP : URMA_CTP, brpc::urma::GetUrmaTpType()); + ASSERT_EQ(rtp ? 0 : 6, brpc::urma::GetUrmaJettyPriority()); + brpc::urma::FLAGS_urma_client_handshake_version = GetParam(); + brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); + } + void TearDown() override { + brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::LEGACY); + } + static bool _mock; +}; +bool UrmaTransportTest::_mock = false; + +TEST_P(UrmaTransportTest, advertises_selected_path_in_both_hello_formats) { + brpc::urma::UrmaEndpoint endpoint(nullptr); + brpc::urma::v2_wire::HelloMessage hello; + endpoint.FillLocalHelloV2(&hello); + EXPECT_EQ(brpc::urma::GetUrmaTpType(), hello.tp_type); + brpc::urma::UrmaHello protobuf_hello; + endpoint.FillLocalHelloV3(&protobuf_hello); + EXPECT_EQ(brpc::urma::GetUrmaTpType(), protobuf_hello.tp_type()); +} + +TEST_P(UrmaTransportTest, exchanges_zero_copy_payloads_and_receive_credits) { + PathEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "single"; + co.timeout_ms = 3000; + co.max_retry = 0; + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + for (int round = 0; round < 40; ++round) { + const std::string payload(round == 0 ? 0 : 20000, 'r'); + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message(payload); + cntl.request_attachment().append(payload); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(payload, response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + } + EXPECT_EQ(before.negotiated_connections + 2, + brpc::urma::GetUrmaRecoveryStatus().negotiated_connections); + server.Stop(0); + server.Join(); +} + +TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_back) { + PathEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 3000; + co.max_retry = 0; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + for (auto mode : {brpc::urma::UrmaRecoveryMode::AUTO, + brpc::urma::UrmaRecoveryMode::TCP_ONLY, + brpc::urma::UrmaRecoveryMode::AUTO}) { + ASSERT_TRUE(brpc::urma::SetUrmaRecoveryMode(mode)); + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + for (int i = 0; i < 3; ++i) { + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message("switch"); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ("switch", response.message()); + } + const auto after = brpc::urma::GetUrmaRecoveryStatus(); + if (mode == brpc::urma::UrmaRecoveryMode::TCP_ONLY) { + EXPECT_EQ(before.negotiated_connections, after.negotiated_connections); + EXPECT_EQ(before.tcp_only_admissions + 3, after.tcp_only_admissions); + } else { + EXPECT_EQ(before.negotiated_connections + 6, after.negotiated_connections); + EXPECT_EQ(before.tcp_only_admissions, after.tcp_only_admissions); + } + } + server.Stop(0); + server.Join(); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaTransportTest, ::testing::Values(2, 3)); + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index fb14511cfc..7372e41c45 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -272,6 +272,76 @@ TEST(UrmaHelperTest, selects_priority_matching_transport_path_type) { EXPECT_EQ(3, urma::FindUrmaPriorityForTpType(attr, URMA_RTP)); EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); + attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( + attr, static_cast(255))); +} + +TEST(UrmaHelperTest, auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); + attr.dev_cap.priority_info[7].tp_type.bs.ctp = 1; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_CTP, path); + EXPECT_EQ(7, priority); +} + +TEST(UrmaHelperTest, explicit_path_uses_matching_priority_and_combined_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[2].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[2].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"rtp", "ctp"}) { + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + EXPECT_EQ(std::string(requested) == "rtp" ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(2, priority); + } +} + +TEST(UrmaHelperTest, rejects_unsupported_paths_in_a_reported_capability_table) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "ctp", &path, &priority)); + EXPECT_EQ(URMA_UTP, path); + EXPECT_EQ(15, priority); + attr.dev_cap.priority_info[0].tp_type.value = 0; + attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); +} + +TEST(UrmaHelperTest, providers_without_path_capabilities_keep_compatibility_priorities) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"auto", "ctp", "rtp"}) { + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + const bool rtp = std::string(requested) == "rtp"; + EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(rtp ? 0 : 6, priority); + } +} + +TEST(UrmaHelperTest, rejects_invalid_path_configuration) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + for (const auto& requested : {"", "utp", "RM", "typo"}) { + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + } + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, nullptr)); } TEST(UrmaHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { @@ -715,6 +785,9 @@ namespace urma { class UrmaRecoveryTestPeer { public: + static int ImportPeer(UrmaEndpoint* ep, const ParsedHello& peer) { + return ep->ImportPeer(peer); + } static void Establish(UrmaEndpoint* ep) { GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); ep->_state = UrmaEndpoint::ESTABLISHED; @@ -833,6 +906,17 @@ TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { "urma_recovery_tcp_fallback_admissions").empty()); } +TEST_F(UrmaRecoveryTest, rejects_mismatched_path_before_importing_resources) { + urma::UrmaEndpoint endpoint(nullptr); + urma::ParsedHello peer; + peer.tp_type = urma::GetUrmaTpType() == URMA_CTP ? URMA_RTP : URMA_CTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); + peer.tp_type = URMA_UTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); +} + TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { From 55aa2873bb014bdf9aa9a0bbe2f57b42764836e8 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:36:00 -0700 Subject: [PATCH 15/29] Fix URMA short connection EOF handling --- src/brpc/urma/urma_endpoint.cpp | 30 +++++++++++++++++---------- src/brpc/urma/urma_endpoint.h | 5 ++++- test/CMakeLists.txt | 3 +++ test/brpc_urma_transport_unittest.cpp | 11 +++++++--- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 6cd1994f1d..a2f163a349 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -1295,8 +1295,6 @@ void UrmaEndpoint::ApplyRemoteHello(const ParsedHello& remote) { // OnNewDataFromTcp: edge-triggered dispatcher. // ============================================================================ -static void TryReadOnTcpDuringUrmaEst(Socket* socket); - void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { auto* tp = static_cast(m->_transport.get()); if (!tp) { @@ -1320,7 +1318,7 @@ void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { if (!IsUrmaAvailable()) { ep->_state = FALLBACK_TCP; tp->_urma_state = UrmaTransport::URMA_OFF; - InputMessenger::OnNewMessages(m); + ep->TryReadOnTcp(); return; } SocketUniquePtr s; @@ -1336,35 +1334,43 @@ void UrmaEndpoint::OnNewDataFromTcp(Socket* m) { } else { s.release(); } - return; } // Client side: handled by ProcessHandshakeAtClient. - return; } else if (state < ESTABLISHED) { // During handshake: wake the handshake bthread parked in ReadFromFd. ep->_read_butex->fetch_add(1, butil::memory_order_release); bthread::butex_wake(ep->_read_butex); - return; } else if (state == FALLBACK_TCP) { - InputMessenger::OnNewMessages(m); + ep->TryReadOnTcp(); return; } else if (state == ESTABLISHED) { - TryReadOnTcpDuringUrmaEst(m); + ep->DrainEstablishedTcp(); return; } + // Return handshake events to the dispatcher as well. Leaving _nevent + // nonzero suppresses later EOF callbacks and retains server resources + // after a short-connection client closes its TCP fd. if (!m->MoreReadEvents(&progress)) { break; } } } -inline void UrmaEndpoint::TryReadOnTcp() { - if (_state.load(butil::memory_order_acquire) == FALLBACK_TCP) { +void UrmaEndpoint::TryReadOnTcp() { + std::unique_lock lock(_tcp_read_mutex); + if (_state.load(butil::memory_order_acquire) == FALLBACK_TCP && + !_socket->Failed()) { InputMessenger::OnNewMessages(_socket); } } -static void TryReadOnTcpDuringUrmaEst(Socket* socket) { +void UrmaEndpoint::DrainEstablishedTcp() { + std::unique_lock lock(_tcp_read_mutex); + if (_state.load(butil::memory_order_acquire) != ESTABLISHED || + _socket->Failed()) { + return; + } + Socket* socket = _socket; int progress = Socket::PROGRESS_INIT; while (true) { uint8_t byte = 0; @@ -1496,6 +1502,7 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { ep->_state = ESTABLISHED; GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); + ep->DrainEstablishedTcp(); return nullptr; } @@ -1592,6 +1599,7 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->_state = ESTABLISHED; GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); + ep->DrainEstablishedTcp(); } else { ep->FallbackToTcp(tp, true); } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index 5d2a3908b0..033f7c9d45 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -262,7 +262,8 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { void PollerAddCqSid(); void PollerRemoveCqSid(); - inline void TryReadOnTcp(); + void TryReadOnTcp(); + void DrainEstablishedTcp(); void FallbackToTcp(UrmaTransport* transport, bool process_tcp); void ReportFailure(UrmaFailureReason reason); void FailHandshake(UrmaTransport* transport, int error, @@ -284,6 +285,8 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { int _handshake_version{0}; // 0 = unnegotiated; 2 = v2; 3 = v3 butil::atomic _pending_received_bytes{0}; butil::Mutex _dispatch_mutex; + // Serialize TCP parsing / EOF checks across event and handshake bthreads. + butil::Mutex _tcp_read_mutex; // The URMA resources (jetty / jfc / jfr / jfce / imported peer objects). UrmaResource* _resource{nullptr}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2a32083158..971fc91752 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -298,6 +298,9 @@ foreach(BRPC_UT ${BRPC_UNITTESTS}) if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_transport_unittest") add_test(NAME brpc_urma_transport_rtp_unittest COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + add_test(NAME brpc_urma_transport_rtp_stress_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp + --gtest_shuffle --gtest_random_seed=92 --gtest_repeat=20) endif() endforeach() diff --git a/test/brpc_urma_transport_unittest.cpp b/test/brpc_urma_transport_unittest.cpp index cfd1dc84e4..94799b372e 100644 --- a/test/brpc_urma_transport_unittest.cpp +++ b/test/brpc_urma_transport_unittest.cpp @@ -147,12 +147,15 @@ TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_bac co.timeout_ms = 3000; co.max_retry = 0; ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + // Cumulative receive buffers exceed the pool capacity unless closed + // short connections release their queues and buffers. + const int requests_per_mode = 40; for (auto mode : {brpc::urma::UrmaRecoveryMode::AUTO, brpc::urma::UrmaRecoveryMode::TCP_ONLY, brpc::urma::UrmaRecoveryMode::AUTO}) { ASSERT_TRUE(brpc::urma::SetUrmaRecoveryMode(mode)); const auto before = brpc::urma::GetUrmaRecoveryStatus(); - for (int i = 0; i < 3; ++i) { + for (int i = 0; i < requests_per_mode; ++i) { brpc::Controller cntl; test::EchoRequest request; test::EchoResponse response; @@ -164,9 +167,11 @@ TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_bac const auto after = brpc::urma::GetUrmaRecoveryStatus(); if (mode == brpc::urma::UrmaRecoveryMode::TCP_ONLY) { EXPECT_EQ(before.negotiated_connections, after.negotiated_connections); - EXPECT_EQ(before.tcp_only_admissions + 3, after.tcp_only_admissions); + EXPECT_EQ(before.tcp_only_admissions + requests_per_mode, + after.tcp_only_admissions); } else { - EXPECT_EQ(before.negotiated_connections + 6, after.negotiated_connections); + EXPECT_EQ(before.negotiated_connections + 2 * requests_per_mode, + after.negotiated_connections); EXPECT_EQ(before.tcp_only_admissions, after.tcp_only_admissions); } } From ed144e7c286e2fe1e20b1c6bec8a398166dec21d Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:07:04 -0700 Subject: [PATCH 16/29] Add opt-in URMA send failure injector for recovery tests --- docs/cn/urma.md | 21 +++++ docs/en/urma.md | 24 +++++ example/urma_performance/CMakeLists.txt | 26 ++++++ .../urma_performance/send_failure_injector.c | 67 ++++++++++++++ .../test_send_failure_injector.c | 90 +++++++++++++++++++ 5 files changed, 228 insertions(+) create mode 100644 example/urma_performance/send_failure_injector.c create mode 100644 example/urma_performance/test_send_failure_injector.c diff --git a/docs/cn/urma.md b/docs/cn/urma.md index edb19c2c96..260353957b 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -251,6 +251,27 @@ cmake --build build/urma-example --target urma_performance_server urma_recovery_ 客户端退出码 0 表示至少一次 RPC 成功;健康基线还需确认 `failure=0`。 功能测试检查模式切换、故障计数及恢复行为,吞吐和 P99 另作性能测试。 +### 定向发送失败测试 + +URMA 示例提供默认关闭的 Linux 注入模块,使用 `LD_PRELOAD` 在指定客户端进程中 +模拟一次数据 SEND 提交失败(EIO),其余调用转发给 liburma,信用 ACK 保持原样。 + +```bash +cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON +cmake --build build/urma-example --target urma_send_failure_injector -j8 +BRPC_URMA_FAIL_ONCE_FILE="$PWD/pr92-inject.once" \ +LD_PRELOAD="$PWD/build/urma-example/liburma_send_failure_injector.so" \ +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ + --mode=auto --connection_type=short --cooldown_ms=10000 --seconds=40 +``` + +启动前移除旧标记文件,正常 RPC 后在另一终端创建 `pr92-inject.once`。 +日志应记录一次注入,失败 RPC 增加一次,`failures.send_post` 增加一次; +冷却期间 `cooled_peers` 和 TCP 准入增加,URMA 协商数暂停,冷却后重新增长。 +该模块验证 SEND_POST 错误处理与恢复策略,设备完成错误需单独注入。 +构建 `urma_send_failure_injector_test` 后,可通过示例目录的 CTest 执行转发与单次注入自测。 + ## 发送访问错误对照 设置 `--urma_copy_send=true` 后,每个数据 WR 使用一个新的、64 字节对齐的注册缓冲区, diff --git a/docs/en/urma.md b/docs/en/urma.md index d56b180ba7..70bc3ce982 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -276,6 +276,30 @@ Exit code 0 means at least one RPC succeeded; a healthy baseline also requires `failure=0`. Functional checks cover mode changes, fault counters and recovery; throughput and P99 belong to a separate performance test. +### Targeted send failure test + +The URMA example provides an optional Linux `LD_PRELOAD` module that rejects one +data SEND with EIO in the selected client process. Other calls, including credit +ACKs, are forwarded to liburma. + +```bash +cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON +cmake --build build/urma-example --target urma_send_failure_injector -j8 +BRPC_URMA_FAIL_ONCE_FILE="$PWD/pr92-inject.once" \ +LD_PRELOAD="$PWD/build/urma-example/liburma_send_failure_injector.so" \ +./build/urma-example/urma_recovery_client --transport=urma \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ + --mode=auto --connection_type=short --cooldown_ms=10000 --seconds=40 +``` + +Remove any old marker before starting, then create `pr92-inject.once` from another +terminal after normal RPCs succeed. Expect one injection log, one failed RPC and +one increment of `failures.send_post`. During cooldown, `cooled_peers` and TCP +admissions increase while URMA negotiations pause; negotiations resume after +cooldown. This tests SEND_POST handling and recovery policy. Device completion +errors require separate injection. Build `urma_send_failure_injector_test` and +run CTest in the example build directory to check forwarding and single injection. + ## Comparing send access failures With `--urma_copy_send=true`, each data WR uses a fresh, 64-byte aligned registered diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 38b58dd6ff..8cd0fa8832 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -21,6 +21,7 @@ project(urma_performance C CXX) include(${CMAKE_CURRENT_LIST_DIR}/../cmake/BrpcExample.cmake) option(LINK_SO "Whether examples are linked dynamically" OFF) +option(BUILD_URMA_FAULT_INJECTOR "Build the process-local SEND failure test module" OFF) brpc_example_find_common_deps(DYNAMIC_LIB) @@ -57,3 +58,28 @@ add_executable(urma_recovery_client ../rdma_performance/recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) brpc_example_configure_target(urma_recovery_client) target_link_libraries(urma_recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +if(BUILD_URMA_FAULT_INJECTOR) + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux") + message(FATAL_ERROR "The URMA fault injector uses Linux LD_PRELOAD") + endif() + add_library(urma_send_failure_injector MODULE send_failure_injector.c) + set_target_properties(urma_send_failure_injector PROPERTIES + C_STANDARD 11 C_STANDARD_REQUIRED ON) + target_include_directories(urma_send_failure_injector PRIVATE ${URMA_INCLUDE_PATH}) + target_link_libraries(urma_send_failure_injector PRIVATE ${CMAKE_DL_LIBS} Threads::Threads) + include(CTest) + if(BUILD_TESTING) + add_library(urma_injector_test_provider SHARED test_send_failure_injector.c) + target_compile_definitions(urma_injector_test_provider PRIVATE URMA_INJECTOR_TEST_PROVIDER) + target_include_directories(urma_injector_test_provider PRIVATE ${URMA_INCLUDE_PATH}) + add_executable(urma_send_failure_injector_test test_send_failure_injector.c) + target_include_directories(urma_send_failure_injector_test PRIVATE ${URMA_INCLUDE_PATH}) + target_link_libraries(urma_send_failure_injector_test PRIVATE urma_injector_test_provider) + add_dependencies(urma_send_failure_injector_test urma_send_failure_injector) + add_test(NAME urma_send_failure_injector_test + COMMAND ${CMAKE_COMMAND} -E env + "LD_PRELOAD=$" + $) + endif() +endif() diff --git a/example/urma_performance/send_failure_injector.c b/example/urma_performance/send_failure_injector.c new file mode 100644 index 0000000000..321fa5c85a --- /dev/null +++ b/example/urma_performance/send_failure_injector.c @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "urma_api.h" + +typedef urma_status_t (*PostSend)(urma_jetty_t*, urma_jfs_wr_t*, urma_jfs_wr_t**); +static PostSend g_post_send; +static pthread_once_t g_resolve_once = PTHREAD_ONCE_INIT; +static atomic_bool g_injected = ATOMIC_VAR_INIT(0); + +static void ResolvePostSend(void) { + g_post_send = (PostSend)dlsym(RTLD_NEXT, "urma_post_jetty_send_wr"); + if (!g_post_send) { + fprintf(stderr, "URMA injector: cannot resolve real post_send: %s\n", dlerror()); + } +} + +// LD_PRELOAD this module into the recovery client. Creating the marker named +// by BRPC_URMA_FAIL_ONCE_FILE rejects one data SEND before it reaches liburma. +// Credit ACKs and all subsequent sends are forwarded to the provider. +urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, + urma_jfs_wr_t** bad_wr) { + pthread_once(&g_resolve_once, ResolvePostSend); + if (!g_post_send) { + if (bad_wr) { + *bad_wr = wr; + } + errno = ENOSYS; + return ENOSYS; + } + const char* marker = getenv("BRPC_URMA_FAIL_ONCE_FILE"); + if (wr && wr->opcode == URMA_OPC_SEND && marker && marker[0] && + access(marker, F_OK) == 0 && !atomic_exchange(&g_injected, 1)) { + if (unlink(marker) != 0) { + fprintf(stderr, "URMA injector: cannot remove marker: %s\n", marker); + } + if (bad_wr) { + *bad_wr = wr; + } + fprintf(stderr, "URMA injector: injected one data SEND post failure, errno=%d\n", EIO); + errno = EIO; + return EIO; + } + return g_post_send(jetty, wr, bad_wr); +} diff --git a/example/urma_performance/test_send_failure_injector.c b/example/urma_performance/test_send_failure_injector.c new file mode 100644 index 0000000000..42269c90b4 --- /dev/null +++ b/example/urma_performance/test_send_failure_injector.c @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include "urma_api.h" + +#ifdef URMA_INJECTOR_TEST_PROVIDER + +static int g_posts; +int UrmaInjectorTestPostCount(void) { return g_posts; } + +urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, + urma_jfs_wr_t** bad_wr) { + (void)jetty; + (void)wr; + if (bad_wr) { + *bad_wr = NULL; + } + ++g_posts; + return URMA_SUCCESS; +} + +#else + +int UrmaInjectorTestPostCount(void); +#define CHECK(expr) do { \ + if (!(expr)) { \ + fprintf(stderr, "Check failed: %s at line %d\n", #expr, __LINE__); \ + return 1; \ + } \ +} while (0) + +int main(void) { + char marker[] = "/tmp/brpc-urma-injector-XXXXXX"; + int fd = mkstemp(marker); + CHECK(fd >= 0); + close(fd); + CHECK(unlink(marker) == 0); + CHECK(setenv("BRPC_URMA_FAIL_ONCE_FILE", marker, 1) == 0); + urma_jfs_wr_t wr = {0}; + urma_jfs_wr_t* bad = NULL; + wr.opcode = URMA_OPC_SEND; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 1); + + FILE* file = fopen(marker, "w"); + CHECK(file != NULL); + fclose(file); + wr.opcode = URMA_OPC_SEND_IMM; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 2); + CHECK(access(marker, F_OK) == 0); + + wr.opcode = URMA_OPC_SEND; + errno = 0; + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == EIO); + CHECK(errno == EIO && bad == &wr && UrmaInjectorTestPostCount() == 2); + CHECK(access(marker, F_OK) != 0); + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 3); + + file = fopen(marker, "w"); + CHECK(file != NULL); + fclose(file); + CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); + CHECK(UrmaInjectorTestPostCount() == 4); + CHECK(unlink(marker) == 0); + puts("PASS: forwarding, credit ACK, single failure, and resumed forwarding"); + return 0; +} + +#endif From a338db6a3b705986549b6ba80b49d140cae8b7be Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:38:56 -0700 Subject: [PATCH 17/29] Synchronize URMA failure injection with RPC readiness --- docs/cn/urma.md | 16 +- docs/en/urma.md | 23 +- example/urma_performance/CMakeLists.txt | 6 + .../run_recovery_injection.py | 209 ++++++++++++++++++ .../test_run_recovery_injection.py | 91 ++++++++ 5 files changed, 324 insertions(+), 21 deletions(-) create mode 100644 example/urma_performance/run_recovery_injection.py create mode 100644 example/urma_performance/test_run_recovery_injection.py diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 260353957b..ac0cd9d118 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -259,18 +259,16 @@ URMA 示例提供默认关闭的 Linux 注入模块,使用 `LD_PRELOAD` 在指 ```bash cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON cmake --build build/urma-example --target urma_send_failure_injector -j8 -BRPC_URMA_FAIL_ONCE_FILE="$PWD/pr92-inject.once" \ -LD_PRELOAD="$PWD/build/urma-example/liburma_send_failure_injector.so" \ -./build/urma-example/urma_recovery_client --transport=urma \ - --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ - --mode=auto --connection_type=short --cooldown_ms=10000 --seconds=40 +python3 example/urma_performance/run_recovery_injection.py \ + --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp ``` -启动前移除旧标记文件,正常 RPC 后在另一终端创建 `pr92-inject.once`。 -日志应记录一次注入,失败 RPC 增加一次,`failures.send_post` 增加一次; -冷却期间 `cooled_peers` 和 TCP 准入增加,URMA 协商数暂停,冷却后重新增长。 +运行器等待成功的 URMA RPC 后创建独立标记文件,直接访问客户端的 loopback 指标端口, +并校验指标 PID。它保存 CSV、日志及基线、冷却中、恢复后三份状态快照, +检查恰好一次失败、`failures.send_post=1`、冷却期间 TCP 准入及冷却后 URMA 协商恢复。 +默认 v3 握手、运行 40 秒、冷却 10 秒;启动等待单独计时。 该模块验证 SEND_POST 错误处理与恢复策略,设备完成错误需单独注入。 -构建 `urma_send_failure_injector_test` 后,可通过示例目录的 CTest 执行转发与单次注入自测。 +构建 `urma_send_failure_injector_test` 后,可通过示例目录的 CTest 执行工具自测。 ## 发送访问错误对照 diff --git a/docs/en/urma.md b/docs/en/urma.md index 70bc3ce982..6c02f569e8 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -285,20 +285,19 @@ ACKs, are forwarded to liburma. ```bash cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON cmake --build build/urma-example --target urma_send_failure_injector -j8 -BRPC_URMA_FAIL_ONCE_FILE="$PWD/pr92-inject.once" \ -LD_PRELOAD="$PWD/build/urma-example/liburma_send_failure_injector.so" \ -./build/urma-example/urma_recovery_client --transport=urma \ - --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ - --mode=auto --connection_type=short --cooldown_ms=10000 --seconds=40 +python3 example/urma_performance/run_recovery_injection.py \ + --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp ``` -Remove any old marker before starting, then create `pr92-inject.once` from another -terminal after normal RPCs succeed. Expect one injection log, one failed RPC and -one increment of `failures.send_post`. During cooldown, `cooled_peers` and TCP -admissions increase while URMA negotiations pause; negotiations resume after -cooldown. This tests SEND_POST handling and recovery policy. Device completion -errors require separate injection. Build `urma_send_failure_injector_test` and -run CTest in the example build directory to check forwarding and single injection. +The runner waits for successful URMA RPCs before creating a unique marker. It +queries the client's loopback metrics directly and checks the listener PID. +CSV, logs and baseline/cooldown/recovered snapshots are saved in a new directory. +Checks require exactly one failed RPC, `failures.send_post=1`, TCP admission during +cooldown and resumed URMA negotiations. Defaults are v3, a 40-second run and +10-second cooldown, with a separate startup timeout. This tests SEND_POST handling +and recovery policy; device completion errors require separate injection. Build +`urma_send_failure_injector_test` and run CTest in the example build directory +for the tool self-tests. ## Comparing send access failures diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 8cd0fa8832..7ae1d26316 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -81,5 +81,11 @@ if(BUILD_URMA_FAULT_INJECTOR) COMMAND ${CMAKE_COMMAND} -E env "LD_PRELOAD=$" $) + find_package(Python3 COMPONENTS Interpreter QUIET) + if(Python3_Interpreter_FOUND) + add_test(NAME urma_recovery_injection_runner_test + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/test_run_recovery_injection.py) + endif() endif() endif() diff --git a/example/urma_performance/run_recovery_injection.py b/example/urma_performance/run_recovery_injection.py new file mode 100644 index 0000000000..e16b7db1d4 --- /dev/null +++ b/example/urma_performance/run_recovery_injection.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Check one SEND failure, peer cooldown, TCP admission and URMA recovery.""" + +import argparse +import csv +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import tempfile +import time +import urllib.request + + +def decode_bvar(text): + match = re.match(r"^\w+\s*:\s*(.*)$", text.strip(), re.DOTALL) + value = match.group(1) if match else text.strip() + try: + result = json.loads(value) + return json.loads(result) if isinstance(result, str) else result + except json.JSONDecodeError: + # Text /vars may quote the JSON object without escaping its quotes. + return json.loads(value[value.index("{"):value.rindex("}") + 1]) + + +def read_samples(path): + with path.open() as stream: + return [list(map(int, row[:1] + row[2:])) for row in csv.reader(stream) + if len(row) == 9 and row[0].isdigit()] + + +def wait_for(proc, predicate, timeout, description): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + value = predicate() + if value: + return value + if proc.poll() is not None: + raise RuntimeError("Client exited while waiting for " + description) + time.sleep(0.1) + raise RuntimeError("Timed out waiting for " + description) + + +def validate_phases(baseline, cooldown, recovered): + if any(s["mode"] != "auto" for s in (baseline, cooldown, recovered)): + raise RuntimeError("Expected AUTO throughout the run") + if len({s["epoch"] for s in (baseline, cooldown, recovered)}) != 1: + raise RuntimeError("Recovery epoch changed during injection") + if (baseline["negotiated_connections"] <= 0 or baseline["cooled_peers"] != 0 + or baseline["failures"]["send_post"] != 0): + raise RuntimeError("Baseline must use URMA with no injected failure") + if (cooldown["failures"]["send_post"] != 1 or cooldown["cooled_peers"] != 1 + or cooldown["peer_cooldown_admissions"] <= 0): + raise RuntimeError("Expected one SEND failure and TCP peer-cooldown admission") + if (recovered["failures"]["send_post"] != 1 or recovered["cooled_peers"] != 0 + or recovered["negotiated_connections"] <= cooldown["negotiated_connections"] + or recovered["peer_cooldown_admissions"] < cooldown["peer_cooldown_admissions"]): + raise RuntimeError("Expected URMA negotiations to resume after cooldown") + + +def run(args): + client = Path(args.client).resolve() + injector = Path(args.injector).resolve() + if not client.is_file() or not injector.is_file(): + raise RuntimeError("Build urma_recovery_client and urma_send_failure_injector first") + output = Path(tempfile.mkdtemp(prefix="pr92-injection-", dir=".")).resolve() + marker = output / "inject.once" + csv_path = output / "client.csv" + log_path = output / "client.log" + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + # All metric requests go directly to this client's loopback listener. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def read_var(name): + request = urllib.request.Request( + "http://127.0.0.1:{}/vars/{}?console=1".format(port, name), + headers={"Accept": "text/plain"}) + with opener.open(request, timeout=2) as response: + text = response.read().decode() + (output / ("latest-" + name + ".txt")).write_text(text) + return decode_bvar(text) + + def record_phase(name, status): + (output / (name + ".json")).write_text(json.dumps(status, indent=2)) + print("{}: send_post={} cooled_peers={} tcp={} negotiated={}".format( + name, status["failures"]["send_post"], status["cooled_peers"], + status["peer_cooldown_admissions"], status["negotiated_connections"]), flush=True) + return status + + command = [str(client), "--transport=urma", "--mode=auto", + "--server=" + args.server, "--urma_device=" + args.device, + "--urma_tp_type=" + args.tp_type, + "--urma_client_handshake_version=" + str(args.handshake_version), + "--urma_buffer_count=4096", "--connection_type=short", + "--interval_ms=200", "--timeout_ms=3000", + "--seconds=" + str(args.seconds), + "--cooldown_ms=" + str(args.cooldown_ms), "--dummy_port=" + str(port)] + env = dict(os.environ, LD_PRELOAD=str(injector), + BRPC_URMA_FAIL_ONCE_FILE=str(marker)) + proc = None + print("Artifacts: " + str(output), flush=True) + try: + with csv_path.open("w") as stdout, log_path.open("w") as stderr: + proc = subprocess.Popen(command, stdout=stdout, stderr=stderr, env=env) + print("Client PID={}; waiting for successful URMA RPCs".format(proc.pid), flush=True) + + def ready(): + rows = read_samples(csv_path) + if rows and rows[-1][2] != 0: + raise RuntimeError("RPC failed before injection") + return rows and rows[-1][1] > 0 and rows[-1][3] > 0 + + wait_for(proc, ready, args.startup_timeout, "successful URMA baseline") + if read_var("pid") != proc.pid: + raise RuntimeError("Metric listener PID does not match the test client") + time.sleep(args.baseline_seconds) + phases = {"baseline": record_phase("baseline", read_var("urma_recovery_status"))} + baseline = phases["baseline"] + if (baseline["failures"]["send_post"] or baseline["cooled_peers"] + or baseline["negotiated_connections"] <= 0): + raise RuntimeError("Unexpected baseline metrics") + marker.touch() + + def cooling(): + status = read_var("urma_recovery_status") + return status if (status["failures"]["send_post"] == 1 + and status["cooled_peers"] == 1 + and status["peer_cooldown_admissions"] > 0) else None + + phases["cooldown"] = record_phase("cooldown", wait_for( + proc, cooling, 5, "peer cooldown and TCP admission")) + + def recovering(): + status = read_var("urma_recovery_status") + return status if (status["cooled_peers"] == 0 and + status["negotiated_connections"] > + phases["cooldown"]["negotiated_connections"]) else None + + phases["recovered"] = record_phase("recovered", wait_for( + proc, recovering, args.cooldown_ms / 1000 + 5, "URMA recovery")) + validate_phases(**phases) + (output / "snapshots.json").write_text(json.dumps( + {"pid": proc.pid, "port": port, "phases": phases}, indent=2)) + rc = proc.wait(timeout=args.seconds + 10) + summary = re.search(r"completed success=(\d+) failure=(\d+)", csv_path.read_text()) + if rc != 0 or not summary or int(summary[1]) == 0 or int(summary[2]) != 1: + raise RuntimeError("Expected exactly one failed RPC and remaining RPCs to succeed") + if log_path.read_text().count("injected one data SEND post failure") != 1: + raise RuntimeError("Expected exactly one injection log") + last = read_samples(csv_path)[-1] + if last[5] != 0 or last[4] != phases["recovered"]["peer_cooldown_admissions"]: + raise RuntimeError("TCP cooldown admissions must stop after URMA recovery") + print(summary[0]) + print("PASS: one SEND failure, TCP cooldown, and URMA recovery") + finally: + if proc is not None and proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--server", required=True) + parser.add_argument("--device", required=True) + parser.add_argument("--client", default="build/urma-example/urma_recovery_client") + parser.add_argument("--injector", default="build/urma-example/liburma_send_failure_injector.so") + parser.add_argument("--tp-type", choices=("rtp", "ctp", "auto"), default="rtp") + parser.add_argument("--handshake-version", type=int, choices=(2, 3), default=3) + parser.add_argument("--seconds", type=int, default=40) + parser.add_argument("--cooldown-ms", type=int, default=10000) + parser.add_argument("--baseline-seconds", type=float, default=5) + parser.add_argument("--startup-timeout", type=float, default=120) + args = parser.parse_args() + if (args.cooldown_ms <= 0 or args.baseline_seconds < 0 or args.startup_timeout <= 0 + or args.seconds < args.baseline_seconds + args.cooldown_ms / 1000 + 5): + parser.error("Duration must cover baseline, cooldown and at least five seconds of recovery") + try: + run(args) + except (OSError, ValueError, KeyError, TypeError, RuntimeError, subprocess.TimeoutExpired) as error: + parser.exit(1, "FAIL: {}\n".format(error)) + + +if __name__ == "__main__": + main() diff --git a/example/urma_performance/test_run_recovery_injection.py b/example/urma_performance/test_run_recovery_injection.py new file mode 100644 index 0000000000..a4991857cb --- /dev/null +++ b/example/urma_performance/test_run_recovery_injection.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest import mock + +sys.dont_write_bytecode = True +import run_recovery_injection as runner + + +class RecoveryInjectionTest(unittest.TestCase): + def test_decode_text_and_json_bvar_formats(self): + value = {"mode": "auto", "failures": {"send_post": 1}} + raw = json.dumps(value) + for body in (raw, 'urma_recovery_status : "' + raw + '"', + "urma_recovery_status : " + json.dumps(raw)): + self.assertEqual(value, runner.decode_bvar(body)) + self.assertEqual(123, runner.decode_bvar("pid : 123\n")) + + def test_samples_skip_headers_summary_and_partial_rows(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "client.csv" + path.write_text("seconds,mode,success,failure\n" + "0,auto,1,0,1,0,0,1,0\n1,auto," + "\ncompleted success=1 failure=0\n") + self.assertEqual([[0, 1, 0, 1, 0, 0, 1, 0]], runner.read_samples(path)) + + def phases(self): + baseline = {"mode": "auto", "epoch": 2, "negotiated_connections": 10, + "cooled_peers": 0, "peer_cooldown_admissions": 0, + "failures": {"send_post": 0}} + cooldown = copy.deepcopy(baseline) + cooldown.update(cooled_peers=1, peer_cooldown_admissions=3) + cooldown["failures"]["send_post"] = 1 + recovered = copy.deepcopy(cooldown) + recovered.update(cooled_peers=0, negotiated_connections=12) + return baseline, cooldown, recovered + + def test_validate_complete_recovery(self): + runner.validate_phases(*self.phases()) + + def test_reject_stale_zero_failure_snapshot(self): + baseline, cooldown, recovered = self.phases() + cooldown["failures"]["send_post"] = 0 + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + + def test_reject_no_recovery_and_epoch_changes(self): + for field, value in (("negotiated_connections", 10), ("epoch", 3)): + baseline, cooldown, recovered = self.phases() + recovered[field] = value + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + + def test_wait_for_readiness_instead_of_fixed_startup_delay(self): + proc = mock.Mock() + proc.poll.return_value = None + ready = mock.Mock(side_effect=[False, False, "ready"]) + with mock.patch.object(runner.time, "sleep"): + self.assertEqual("ready", runner.wait_for(proc, ready, 1, "baseline")) + self.assertEqual(3, ready.call_count) + + def test_exit_before_readiness_is_a_failure(self): + proc = mock.Mock() + proc.poll.return_value = 1 + with self.assertRaises(RuntimeError): + runner.wait_for(proc, lambda: False, 1, "baseline") + + +if __name__ == "__main__": + unittest.main() From f8eaa64aae7a504e912551a2a8ad6de0b7ba7479 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:54:14 -0700 Subject: [PATCH 18/29] Condense runtime recovery documentation --- docs/cn/rdma.md | 6 +- docs/cn/rdma_runtime_recovery.md | 162 ++++++------------------ docs/cn/urma.md | 141 ++++++++------------- docs/en/urma.md | 181 ++++++++++----------------- example/rdma_performance/RECOVERY.md | 62 ++++----- 5 files changed, 189 insertions(+), 363 deletions(-) diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index 052fbc13d0..7dd2689156 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,7 +1,7 @@ -# 运行期故障恢复设计 +# 运行期故障恢复 -端口故障隔离、冷却探测、运行时降级及多端口主备方案见 -[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md)。 +模式切换、端口与对端冷却及 TCP 回落见 +[RDMA 与 URMA 运行期故障恢复](rdma_runtime_recovery.md)。 # 编译 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index d2141c89a5..0a2f71e313 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -17,144 +17,62 @@ specific language governing permissions and limitations under the License. --> -# RDMA 运行期故障恢复与 TCP 降级设计 +# RDMA 与 URMA 运行期故障恢复 -基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 +运行期故障后,恢复策略控制后续连接的传输选择: -## 首版实现 +故障上报 → 对端或端口冷却 → 新连接使用 TCP → 冷却结束后恢复准入。 -`RdmaRecoveryPolicy` 已接入客户端和服务端握手入口,控制新连接的 RDMA 准入: +## 模式 -- LEGACY 为默认模式,沿用现有行为;TCP_ONLY 让新连接使用 TCP。 -- AUTO 下,客户端收到 RETRY_EXC / RNR_RETRY_EXC 后,按远端地址冷却 5 秒。 -- 当前设备的 PORT_ERR 阻止新 RDMA 建链,端口恢复且冷却结束后重新准入。 -- 对端冷却缓存最多 1024 项,满时淘汰最早到期项;模式切换用 epoch 过滤旧连接错误。 -- 故障上报覆盖 QP/CQ 异步错误、CQ 轮询/事件/通知错误及链路类发送错误,同一连接只计一次。 -- QP/CQ 绑定所属 socket,异步事件核对当前资源;发生运行期错误的资源直接释放。 - -`GetRdmaRecoveryStatus()` 和 `/vars/rdma_recovery_*` 暴露冷却状态、分原因故障计数、 -TCP 回落准入数及 RDMA 协商数。实机步骤见 -[RDMA 双机故障恢复测试](../../example/rdma_performance/RECOVERY.md)。 - -通过进程级 API 配置: - -```cpp -#include "brpc/rdma/rdma_recovery_policy.h" - -brpc::rdma::SetRdmaRecoveryCooldownMs(5000); -brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::AUTO); -// 运维降级 -brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::TCP_ONLY); -``` - -当前模式切换作用于新建连接,存量连接继续使用已协商的传输。 -冷却到期后重新开放准入,TCP 连接沿用现有复用与重建机制。 -后续迭代增加存量连接退休、单连接探测、指数退避、发送 watchdog 和多端口主备。 - -## URMA 适配 - -URMA 复用准入算法,独立维护模式、epoch、对端缓存和 `urma_recovery_*` 指标。 -接入完成错误、JFC 轮询/事件/通知错误及 WR 提交失败,支持 v2/v3 服务端协商降级。 -配置与行为见 [URMA 运行期故障恢复](urma.md#运行期故障恢复)。 - -以下为完整方案。 - -## 当前问题 - -RDMA 握手阶段支持回落 TCP,建链后的错误 CQE 主要通过 `Socket::SetFailed` 处理。 -`PORT_ERR` / `PORT_ACTIVE` 事件仅做确认,缺少端口隔离、冷却和恢复探测。 -`GlobalDisableRdma` 使用 FATAL,需要独立的运行时降级入口。 - -相关代码:`src/brpc/rdma/rdma_endpoint.cpp`、`src/brpc/rdma/rdma_helper.cpp`、 -`src/brpc/rdma_transport.cpp`。 - -## 恢复流程 - -故障检测 → 隔离故障路径 → 新连接选择备用 RDMA 或 TCP → 冷却 → 单连接探测 → 恢复。 - -新增 `RdmaRecoveryPolicy`,分别管理本地端口和对端路径的健康状态。 -本地端口以 device、port 标识,对端路径增加远端地址。 -`RdmaTransport` 建链时获取路由快照,`RdmaEndpoint` 持有对应设备资源。 +| 模式 | 新连接行为 | +| --- | --- | +| LEGACY(默认) | 沿用原有握手和错误处理 | +| AUTO | 按端口和对端状态准入,冷却期间选择 TCP | +| TCP_ONLY | 直接选择 TCP | -发生故障时,先发布隔离状态,再使受影响连接失败并唤醒写等待者。 -后续连接按主 RDMA、备用 RDMA、TCP 的顺序选择可用路径。 -TCP 降级使用新连接;存量 RPC 交给现有 Controller 重试策略处理,沿用原配额和 deadline。 +模式切换作用于新连接,存量连接继续使用已协商的传输。 +当前 RPC 沿用 Controller 的重试次数和 deadline。 -## 状态与错误处理 +## 冷却与错误处理 -路径状态为 HEALTHY、COOLDOWN、PROBING、DISABLED。 +默认冷却 5 秒,使用单调时钟。客户端按服务端 IP 和端口记录对端状态, +缓存上限为 1024 项,满时淘汰最早到期项。RDMA 和 URMA 独立维护状态与指标, +模式切换递增 epoch,过滤旧连接的故障上报。 | 事件 | 处理 | | --- | --- | -| PORT_ERR | 端口进入 COOLDOWN,关闭该端口上的连接 | -| PORT_ACTIVE | 标记端口可探测,冷却结束后进入 PROBING | -| QP/CQ 错误、RETRY_EXC | 连接失败并隔离对应对端路径 | -| RNR_RETRY_EXC | 隔离对端路径,记录接收资源或流控异常 | -| SQ/RQ 窗口耗尽 | 返回 EAGAIN,等待 completion 或 credit | -| post_send 失败 | 保存 errno,使连接失败并按错误类型更新路径状态 | -| DEVICE_FATAL | 禁用对应设备资源域,沿用现有 FATAL 处理 | - -冷却使用单调时钟,连续失败采用指数退避和 jitter。 -到期后最多准入一个 RDMA 探测连接,其余新连接继续走备用路径或 TCP。 -探测使用正常业务流量;收到有效 RDMA 数据且发送获得完成后恢复 HEALTHY, -失败或探测超时则重新冷却。 - -每次状态更新携带 generation 和 policy epoch。 -同一连接代际只报告一次故障,晚到事件按代际过滤。 - -## 运行时降级 - -提供进程级 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 和 setter/getter。 -LEGACY 为默认值;AUTO 启用恢复策略;TCP_ONLY 将后续请求切到 TCP。 - -模式更新递增 policy epoch。TCP_ONLY 下,健康存量 RPC 完成后连接退休; -请求获取 socket 时检查退休状态,新请求使用新连接。 -切回 AUTO 后进入 PROBING,逐步恢复 RDMA。 - -SocketMap 和连接池按传输模式、设备资源域和 epoch 区分连接。 -socket 复活时重新获取路由快照。服务实例的健康检查与 RDMA 路径状态分开维护。 - -## 多端口主备 +| RDMA PORT_ERR | 暂停当前端口的新 RDMA 连接,记录端口冷却 | +| RDMA PORT_ACTIVE | 更新端口状态,冷却结束后恢复准入 | +| 重试耗尽、RNR、QP/CQ/JFC 错误、WR 提交失败 | 上报故障,AUTO 下冷却客户端对端 | +| 访问错误 | 单独计数 | +| WR flush | 沿用已记录的根因 | +| SQ 或接收 credit 耗尽 | 返回 EAGAIN,等待 completion 或 credit | -启动时配置主备路径,切换时创建新 QP 并交换对应 GID/QPN。 -同 HCA 多端口共享 PD 时复用注册内存,各端口独立维护路由和健康状态。 +同一连接只记录首个运行期故障。QP/CQ 异步事件核对资源归属,故障资源释放。 +URMA 握手支持 v2/v3;短连接关闭后回收队列和接收缓冲,TCP 解析与关闭检查串行执行。 -跨 HCA 引入 `RdmaDeviceContext`,分别持有 context、PD、事件源和 MR 注册表。 -IOBuf 和用户内存按目标 PD 查找 lkey,并管理各设备上的注册与引用生命周期。 -备用设备注册失败时选择 TCP。 +## 配置 -回收顺序为:停止准入和 WR 提交,停止 CQ 处理,reset/drain QP/CQ, -再释放 buffer、MR、PD 和 context。停机先取消探测与 watchdog,再等待回调退出。 - -## SQ 停顿与 RNR - -正常窗口耗尽保持 EAGAIN 背压。发送 watchdog 跟踪 outstanding WR、 -待发送数据和 credit 的进展;超过阈值后使连接失败并隔离对端路径。 -计时从发送受阻开始,发送完成或 credit 前进时更新。 - -ACK timeout、retry count 和 RNR retry 保持当前默认值,支持配置有限重试次数。 -RNR 重试耗尽作为对端路径故障处理,发送停顿检测独立于 RPC deadline。 +```cpp +#include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" -## 配置与指标 +brpc::rdma::SetRdmaRecoveryCooldownMs(5000); +brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::AUTO); -| 参数 | 建议默认值 | 用途 | -| --- | --- | --- | -| rdma_recovery_mode | legacy | 选择恢复模式 | -| rdma_path_cooldown_ms | 5000 | 初始冷却时间 | -| rdma_path_max_cooldown_ms | 60000 | 退避上限 | -| rdma_recovery_probe_timeout_ms | 1000 | 探测预算,受调用 deadline 约束 | -| rdma_send_stall_timeout_ms | 0 | 发送停顿阈值,0 关闭 | -| rdma_backup_paths | 空 | 启动时配置主备路径 | +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` -指标覆盖端口状态、隔离次数、TCP 降级次数、探测结果、旧 epoch 活动连接数和发送停顿时长。 -日志记录 device、port、socket generation、WC status/vendor_err 和 errno。 +运行时调用对应 setter 切换 `LEGACY`、`AUTO` 或 `TCP_ONLY`。 +URMA 使用 CTP/RTP 路径,设备选择见 [URMA 配置](urma.md#传输路径选择)。 -## 实现顺序与验证 +## 指标与验证 -1. 实现单设备故障分类、端口隔离、冷却探测、TCP 降级和运行时模式切换。 -2. 增加发送停顿检测、有限 RNR 重试配置和诊断指标。 -3. 拆分设备资源域,支持多端口及跨 HCA 主备。 +`GetRdmaRecoveryStatus()`、`GetUrmaRecoveryStatus()` 及 `/vars/rdma_recovery_*`、 +`/vars/urma_recovery_*` 提供模式、epoch、分原因故障计数、冷却状态、 +传输协商数和 TCP 准入数。TCP 准入按连接累计。 -单测覆盖冷却与并发探测、错误分类、窗口恢复、模式切换、连接复用和代际回收。 -双端 Linux RDMA 测试覆盖断链恢复、RNR 和发送停顿;双 HCA 测试验证主备切换与 MR 注册。 -对比 LEGACY、健康 AUTO 和降级 TCP 的恢复时延、吞吐、P99 和 CPU。 +- [RDMA 双机测试](../../example/rdma_performance/RECOVERY.md):模式切换、端口和对端故障。 +- [URMA 定向注入](urma.md#定向发送失败测试):一次 SEND 提交失败、TCP 冷却和 URMA 恢复。 diff --git a/docs/cn/urma.md b/docs/cn/urma.md index ac0cd9d118..c7df2ad228 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -20,22 +20,17 @@ WR。完成事件既可由 JFC 忙轮询获取,也可通过 JFCE 事件 fd 获 ### CMake 编译 ```bash -# 带 URMA 支持编译 brpc -cmake -B build -DWITH_URMA=ON -make -C build -j$(nproc) - -# 编译 urma_performance 示例 -cd example/urma_performance -cmake -B build -make -C build -j$(nproc) +cmake -S . -B build -DWITH_URMA=ON +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/urma_performance -B build/urma-example \ + -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/urma-example -j8 ``` -`WITH_URMA=ON` 使用上游 UMDK 头文件进行编译。CMake 优先使用系统安装的 -SDK;找不到头文件时,会参照 Mooncake 的 mock 构建方式下载固定版本的 -UMDK commit,可通过 `DOWNLOAD_URMA_HEADERS=OFF` 禁止下载。找到 `liburma` 时使用 -真实硬件数据通路,否则链接 brpc 的 mock,使 URMA 代码和测试仍可在无硬件 -环境编译。`URMA_USE_MOCK` 默认为 `AUTO`;设为 `ON` 可强制使用 mock,设为 -`OFF` 则要求找到 `liburma` 并排除 mock。 +CMake 优先使用系统 UMDK 头文件,缺失时下载固定版本; +`DOWNLOAD_URMA_HEADERS=OFF` 关闭下载。`URMA_USE_MOCK=AUTO` 按 `liburma` 选择实现, +`ON` 使用 mock,`OFF` 使用硬件库。示例通过 `URMA_ROOT` 或 `URMA_INCLUDE_PATH` 指定 SDK 头文件。 ### Make 编译 @@ -60,7 +55,7 @@ sh config_brpc.sh --headers=/usr/include --libs=/usr/lib \ # 编译带 URMA 支持的 brpc bazel build --define=BRPC_WITH_URMA=true //:brpc -# 仅展示关闭下载的开关;头文件仍须由工具链或本地 BUILD 目标声明 +# 使用本地声明的 SDK 头文件 bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc @@ -73,14 +68,9 @@ bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_URMA_USE_MOCK=false //:brpc ``` -Bazel 会自动获取固定 commit 的 UMDK 头文件,并通过 `@umdk//:urma_headers` -提供给 URMA 目标。设置 `BRPC_DOWNLOAD_URMA_HEADERS=false` 后,Bazel 不再依赖 -该外部仓库;实际构建仍需通过工具链或本地 BUILD 目标声明 -`urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。仅通过 -`--copt=-I/path` 指向工作区外目录,在默认 Bazel 沙箱下不会把未声明的头文件 -带入编译输入。设置 -`BRPC_URMA_USE_MOCK=true` 强制加入 mock;设置为 `false` 则排除 mock 并链接 -`liburma`。未指定时,URMA 构建默认使用 mock。 +Bazel 通过 `@umdk//:urma_headers` 使用固定版本的 UMDK 头文件。 +关闭下载时,由工具链或本地 BUILD 目标声明 `urma_api.h`、`urma_types.h` 和 `urma_ubagg.h`。 +`BRPC_URMA_USE_MOCK=true` 使用 mock,`false` 链接 `liburma`;默认使用 mock。 ## 使用 @@ -130,19 +120,15 @@ urma::UrmaEndpoint : public SocketUser (urma/urma_endpoint.{h,cpp}) 3. 双方在 TCP fd 上交换 `UrmaHello` 消息(v2 二进制 magic `URMA`, v3 protobuf magic `URM3`),携带本地 EID、jetty id、recv buffer 数量、 以及扁平化的 buffer 池 segment。 -4. 双方先调用 `urma_import_seg` **再**调用 `urma_import_jetty`,为远端 EID - 建立传输路径(TP)路由。跳过 `import_seg` 会导致首个 SEND 被硬件以 - `URMA_CR_RNR_RETRY_CNT_EXC_ERR` 拒绝。 +4. 双方依次调用 `urma_import_seg` 和 `urma_import_jetty`,建立远端路由。 5. 4 字节 ACK(`HELLO_ACK_URMA_OK = 0x1`)确认双方均要 URMA。 6. 成功后 TCP fd 仅保留用于 epoll 生命周期和回退,数据走 URMA。 ### 内存管理 -申请一大段 `mmap` 内存,用 `urma_register_seg` 一次性注册,再切成固定大小 -buffer(默认 8KB)。劫持 `butil::iobuf::blockmem_allocate` 使每个 IOBuf -block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 构建 -`urma_sge_t`,无需逐消息注册(与 RDMA `block_pool` 设计一致)。用户注册 -内存通过 `urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 支持。 +通过 `mmap` 分配内存,调用 `urma_register_seg` 注册后切为固定大小的缓冲区(默认 8KB)。 +IOBuf 使用注册池分配器,发送路径从 block refs 构建 `urma_sge_t`。 +用户内存通过 `RegisterMemoryForUrma` / `DeregisterMemoryForUrma` 管理。 ## 配置 @@ -174,6 +160,15 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs 前配置 provider。默认 standalone+IODIE 配置与 UMDK 性能工具保持一致。 bonding 支持需要 provider 扩展头文件 `urma_ubagg.h`。 +### 传输路径选择 + +`--urma_tp_type=auto` 按设备能力优先选择 CTP,其次 RTP;`ctp`、`rtp` 显式指定路径。 +两端使用相同路径,进程初始化后保持固定。Jetty 取首个匹配优先级,支持组合能力位; +设备映射决定 SL,例如 priority 0 映射 SL 2。日志记录实际 `tp_type` 和 `priority`。 + +设备未提供能力表时,auto/ctp 使用 CTP priority 6,rtp 使用 RTP priority 0; +路径选择失败时初始化回落 TCP。 + ## 与 UBRing 协同 UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing @@ -187,33 +182,19 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing | 跨节点大包(>= 64KB) | `SOCKET_MODE_URMA` | | 传统 RoCE/IB 数据中心 | `SOCKET_MODE_URMA` 或 `SOCKET_MODE_RDMA` | -单连接内按包大小自动分流的方案(路线 C 方案 B)作为后续演进方向。 - -## 限制 +## 运行环境 -- 仅支持 `baidu_std` 协议(与 RDMA 一致)。SSL、RTMP、NSHEAD、MONGO 在 - `ContextInitOrDie` 阶段拒绝。 -- 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 -- 当前实现面向 Linux。 - -### 传输路径选择 - -进程初始化时,`--urma_tp_type=auto` 从设备表中选择第一个支持 CTP 的优先级, -没有 CTP 时选择第一个支持 RTP 的优先级,同时支持组合能力位。 -v2/v3 握手携带所选路径,Jetty 使用对应优先级。两端需配置相同路径, -不匹配的对端路径会在导入资源前被拒绝。路径在进程生命周期内保持固定。 - -仅提供 RTP 的设备,两端均使用 `--urma_device=DEVICE --urma_tp_type=rtp`。 -优先级是设备表索引,映射的 SL 可以不同,例如 priority 0 映射 SL 2。 -brpc 按路径能力选择优先级,设备映射保持原值。初始化日志记录 `tp_type` 和 `priority`。 - -能力表已报告但不支持显式指定路径时,初始化沿用现有 TCP 回落流程。 -未报告任何路径能力的 provider 保留兼容值:auto/ctp 使用 CTP priority 6, -rtp 使用 RTP priority 0。验收 URMA 数据通路时检查协商计数。 +Linux、`baidu_std` 协议,以及支持目标设备的 UMDK provider 和 `liburma`。 ## 运行期故障恢复 -通过进程级 API 控制新连接的传输选择: +恢复策略按进程配置,作用于新连接: + +| 模式 | 行为 | +| --- | --- | +| LEGACY(默认) | 沿用原有握手和错误处理 | +| AUTO | 对端故障后冷却,期间使用 TCP,到期恢复 URMA 准入 | +| TCP_ONLY | 直接使用 TCP | ```cpp #include "brpc/urma/urma_recovery_policy.h" @@ -222,39 +203,30 @@ brpc::urma::SetUrmaRecoveryCooldownMs(5000); brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); ``` -默认 LEGACY 沿用现有行为;AUTO 在重试耗尽、RNR、JFC 错误或 WR 提交失败后, -将客户端对端地址冷却 5 秒,期间新连接使用 TCP。TCP_ONLY 让新连接直接选择 TCP。 -服务端通过 v2/v3 hello 拒绝 URMA 协商,随后处理 TCP 请求。 +默认冷却 5 秒,故障来源包括重试耗尽、RNR、JFC 错误和 WR 提交失败。 +同一连接记录首个运行期故障;访问错误单独计数,窗口耗尽保持 EAGAIN。 +URMA 与 RDMA 独立维护模式、epoch 和冷却缓存,模式切换过滤旧连接上报。 -URMA 和 RDMA 分别维护模式、epoch、冷却缓存和指标。同一连接只上报首个运行期故障; -模式切换过滤旧 epoch 的错误。访问错误单独计数,WR flush 不重复归因, -SQ/接收 credit 耗尽保留 EAGAIN 背压,ACK 提交失败向连接错误处理传播。 +`GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态及协商数; +`/vars/urma_recovery_tcp_fallback_admissions` 按连接累计 TCP 准入。 -`GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态和协商数, -`/vars/urma_recovery_tcp_fallback_admissions` 提供 TCP 准入次数。 -模式切换作用于新连接;存量连接使用已协商的传输,冷却到期后重新准入 URMA。 +### 功能验证 -`example/urma_performance` 构建 `urma_recovery_client`,复用 RDMA 示例的恢复客户端源码, -通过 URMA 示例链接依赖,支持仅启用 URMA 的构建。增加 `--transport=urma` 选择 URMA, -使用 `--mode`、`--mode_file` 和 CSV 统计验证双机功能: +使用上文构建的 `urma_performance_server` 和 `urma_recovery_client`,两端指定相同设备路径。 +客户端示例: ```bash -cmake --build build/urma-example --target urma_performance_server urma_recovery_client -j8 ./build/urma-example/urma_recovery_client --transport=urma \ - --server=SERVER_IP:8003 --urma_device=URMA_DEVICE \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ --mode=auto --connection_type=short --seconds=15 --interval_ms=200 ``` -`build/urma-example` 使用本文编译配置生成,链接当前 URMA brpc 静态库。 -先分别运行 `auto` 与 `tcp_only`,检查 RPC 成功数、URMA 协商数和 TCP 准入数。 -运行中通过 `--mode_file` 指向的文件切换 `auto`、`tcp_only`,使用 short 连接观察新准入。 -客户端退出码 0 表示至少一次 RPC 成功;健康基线还需确认 `failure=0`。 -功能测试检查模式切换、故障计数及恢复行为,吞吐和 P99 另作性能测试。 +健康 AUTO 检查 `failure=0`、URMA 协商增加、TCP 准入为 0; +TCP_ONLY 检查 `failure=0`、TCP 准入增加。运行中通过 `--mode_file` 切换模式。 ### 定向发送失败测试 -URMA 示例提供默认关闭的 Linux 注入模块,使用 `LD_PRELOAD` 在指定客户端进程中 -模拟一次数据 SEND 提交失败(EIO),其余调用转发给 liburma,信用 ACK 保持原样。 +启用 Linux 注入模块,在指定客户端模拟一次数据 SEND 提交失败(EIO): ```bash cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON @@ -263,16 +235,13 @@ python3 example/urma_performance/run_recovery_injection.py \ --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp ``` -运行器等待成功的 URMA RPC 后创建独立标记文件,直接访问客户端的 loopback 指标端口, -并校验指标 PID。它保存 CSV、日志及基线、冷却中、恢复后三份状态快照, -检查恰好一次失败、`failures.send_post=1`、冷却期间 TCP 准入及冷却后 URMA 协商恢复。 -默认 v3 握手、运行 40 秒、冷却 10 秒;启动等待单独计时。 -该模块验证 SEND_POST 错误处理与恢复策略,设备完成错误需单独注入。 -构建 `urma_send_failure_injector_test` 后,可通过示例目录的 CTest 执行工具自测。 +运行器等待 URMA RPC 成功后注入,校验指标 PID,保存 CSV、日志和三阶段快照。 +默认 v3 握手、运行 40 秒、冷却 10 秒;验收一次预期失败、`send_post=1`、 +冷却期间 TCP 准入增加、冷却后 URMA 协商恢复。 +构建 `urma_send_failure_injector_test` 后,通过示例目录的 CTest 执行工具自测。 -## 发送访问错误对照 +### 发送访问错误对照 -设置 `--urma_copy_send=true` 后,每个数据 WR 使用一个新的、64 字节对齐的注册缓冲区, -该缓冲区由发送槽位持有至完成;消息和附件内容保持一致。此模式增加一次 CPU 拷贝。 -可在两端保持设备、队列、消息大小等参数一致,比较原发送路径与复制路径的结果。 -启动日志显示 `copy_send`,失败日志显示完成方向及发送槽位的地址、长度和注册段范围。 +`--urma_copy_send=true` 将每个数据 WR 合并到 64 字节对齐的注册缓冲区,并持有至完成, +增加一次 CPU 拷贝。两端保持设备、队列和消息大小一致,对比原发送路径与复制路径。 +初始化日志记录 `copy_send`,失败日志记录完成方向、槽位及注册段范围。 diff --git a/docs/en/urma.md b/docs/en/urma.md index 6c02f569e8..5857b272ec 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -20,24 +20,18 @@ from a JFC either by busy polling or through a JFCE event fd. ### Build with CMake ```bash -# Build brpc with URMA support -cmake -B build -DWITH_URMA=ON -make -C build -j$(nproc) - -# Build the urma_performance example -cd example/urma_performance -cmake -B build -make -C build -j$(nproc) +cmake -S . -B build -DWITH_URMA=ON +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/urma_performance -B build/urma-example \ + -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/urma-example -j8 ``` -`WITH_URMA=ON` compiles against upstream UMDK headers. CMake prefers an -installed SDK and, following Mooncake's mock setup, downloads UMDK at a pinned -commit when the headers are unavailable. Set `DOWNLOAD_URMA_HEADERS=OFF` to -disable downloading. -When `liburma` is found it is linked for the hardware data path. Otherwise, -brpc uses its link-time mock so URMA code and tests can still be built without -hardware. `URMA_USE_MOCK` defaults to `AUTO`; set it to `ON` to force the mock, -or to `OFF` to require `liburma` and exclude the mock. +CMake uses installed UMDK headers or downloads a pinned version. +`DOWNLOAD_URMA_HEADERS=OFF` disables downloading. `URMA_USE_MOCK=AUTO` selects by +`liburma` availability; `ON` uses the mock and `OFF` uses the hardware library. +Set `URMA_ROOT` or `URMA_INCLUDE_PATH` to select SDK headers for the example. ### Build with Make @@ -63,7 +57,7 @@ requires `liburma` in the configured library paths. # Build brpc with URMA support bazel build --define=BRPC_WITH_URMA=true //:brpc -# This only shows the download switch; headers must still be declared by the toolchain or a local BUILD target +# Use locally declared SDK headers bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_DOWNLOAD_URMA_HEADERS=false //:brpc @@ -76,15 +70,10 @@ bazel build --define=BRPC_WITH_URMA=true \ --define=BRPC_URMA_USE_MOCK=false //:brpc ``` -Bazel fetches the pinned UMDK commit and exposes its headers through -`@umdk//:urma_headers` by default. With -`BRPC_DOWNLOAD_URMA_HEADERS=false`, Bazel does not depend on that external -repository; the actual build must still declare `urma_api.h`, `urma_types.h`, -and `urma_ubagg.h` through the toolchain or a local BUILD target. Passing only -`--copt=-I/path` to a directory outside the workspace does not make undeclared -headers available inside Bazel's default sandbox. -`BRPC_URMA_USE_MOCK=true` forces the link-time mock, while `false` excludes it -and links `liburma`. If unspecified, URMA builds use the mock by default. +Bazel uses pinned UMDK headers through `@umdk//:urma_headers`. +With downloading disabled, declare `urma_api.h`, `urma_types.h` and `urma_ubagg.h` +through the toolchain or a local BUILD target. `BRPC_URMA_USE_MOCK=true` uses the +mock; `false` links `liburma`. The default is the mock. ## Usage @@ -137,23 +126,17 @@ Like RDMA / UBRing, the control plane is TCP and the data plane is URMA: 3. Both sides exchange a `UrmaHello` message (magic `URMA` for v2 binary, `URM3` for v3 protobuf) over the TCP fd. The message carries the local EID, jetty id, recv buffer count, and the flattened buffer-pool segment. -4. Each side calls `urma_import_seg` **before** `urma_import_jetty` to - establish transport-path (TP) routing for the remote EID. Skipping the - `import_seg` step causes the first SEND to be rejected by hardware with - `URMA_CR_RNR_RETRY_CNT_EXC_ERR`. +4. Each side calls `urma_import_seg`, then `urma_import_jetty`, to establish the remote route. 5. A 4-byte ACK (`HELLO_ACK_URMA_OK = 0x1`) confirms both sides want URMA. 6. On success, the TCP fd is kept only for the epoll lifecycle and fallback; payloads flow through URMA. ### Memory management -A single large region is `mmap`-ed and registered once with -`urma_register_seg`, then sliced into fixed-size buffers (default 8 KB). -`butil::iobuf::blockmem_allocate` is hijacked so every IOBuf block is backed -by the registered segment, allowing the send path to build `urma_sge_t` -directly from IOBuf block refs without per-message registration (mirroring the -RDMA `block_pool` design). User-registered memory is supported via -`urma::RegisterMemoryForUrma` / `DeregisterMemoryForUrma`. +A region allocated with `mmap` is registered with `urma_register_seg` and split +into fixed-size buffers (default 8 KB). IOBuf uses the registered-pool allocator; +the send path builds `urma_sge_t` from block refs. Manage user memory with +`RegisterMemoryForUrma` / `DeregisterMemoryForUrma`. ## Configuration @@ -186,6 +169,17 @@ immediately after context creation and before creating segments or queues. The default standalone+IODIE combination matches the UMDK performance tool. Bonding support requires the provider extension header `urma_ubagg.h`. +### Transport path selection + +`--urma_tp_type=auto` prefers CTP, then RTP, from device capabilities; `ctp` and +`rtp` select a path explicitly. Both peers use the same path, fixed at process +initialization. Jettys use the first matching priority, including combined +capability bits. The device maps priority to SL, such as priority 0 to SL 2. +Logs report the selected `tp_type` and `priority`. + +Providers without a capability table use CTP priority 6 for auto/ctp or RTP +priority 0 for rtp. Path selection failure uses initialization's TCP fallback. + ## Coexistence with UBRing UrmaTransport is the recommended transport for **large packets and @@ -200,39 +194,19 @@ syscalls). For a mixed workload, select the transport per service: | Cross-node large packets (>= 64 KB) | `SOCKET_MODE_URMA` | | Traditional RoCE / IB datacenter | `SOCKET_MODE_URMA` or `SOCKET_MODE_RDMA` | -A single-connection hybrid that auto-routes by packet size (Route C, scheme B) -is tracked as a future enhancement. +## Runtime environment -## Limitations - -- `baidu_std` protocol only (same as RDMA). SSL, RTMP, NSHEAD, MONGO are - rejected at `ContextInitOrDie`. -- A hardware data path requires a supported UMDK provider and `liburma`. -- The current implementation targets Linux. - -### Transport path selection - -At process initialization, `--urma_tp_type=auto` selects the first priority -advertising CTP, or the first RTP priority when CTP is absent. Combined capability -bits are supported. Both handshake versions advertise the selected path, and -Jettys use its priority. Configure both peers for the same path; an incompatible -peer path is rejected before importing resources. The path remains fixed for -the process lifetime. - -For an RTP-only device, start both the server and client with -`--urma_device=DEVICE --urma_tp_type=rtp`. A priority value is a device table -index and can map to a different SL. For example, priority 0 may map to SL 2. -brpc selects the priority from the path capability table and leaves the mapping -unchanged. Initialization logs report `tp_type` and `priority`. - -An explicit path missing from a reported capability table disables URMA through -the existing initialization fallback. Providers reporting no path capabilities -retain compatibility values: CTP priority 6 for auto/ctp, RTP priority 0 for rtp. -Check negotiation counters when validating a URMA data path. +Linux, the `baidu_std` protocol, and a UMDK provider with `liburma` for the target device. ## Runtime recovery -Configure admission for new connections with the process-level API: +Configure recovery per process for new connections: + +| Mode | Behavior | +| --- | --- | +| LEGACY (default) | Existing handshake and error handling | +| AUTO | Cool a failed peer, use TCP during cooldown, then resume URMA admission | +| TCP_ONLY | Select TCP | ```cpp #include "brpc/urma/urma_recovery_policy.h" @@ -241,46 +215,34 @@ brpc::urma::SetUrmaRecoveryCooldownMs(5000); brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); ``` -LEGACY is the default. AUTO cools a client peer for five seconds after retry -exhaustion, RNR, JFC errors or WR submission failures. New connections use TCP -during cooldown. TCP_ONLY selects TCP for new connections. Servers decline -URMA through the v2/v3 hello and continue processing TCP requests. +The default cooldown is five seconds. Sources include retry exhaustion, RNR, +JFC errors and WR submission failures. Each connection records its first runtime +failure; access errors have a separate counter, and exhausted windows return EAGAIN. +URMA and RDMA keep separate modes, epochs and cooldown caches. Mode changes filter +reports from older connections. -URMA and RDMA keep separate modes, epochs, cooldown caches and metrics. -Each connection reports its first runtime failure; mode changes filter reports -from older epochs. Access errors are counted separately, flush completions do -not add a root cause, and exhausted SQ/receive credits retain EAGAIN backpressure. -ACK submission failures propagate to connection error handling. +`GetUrmaRecoveryStatus()` and `/vars/urma_recovery_status` expose failure reasons, +cooldown state and negotiation counts. `/vars/urma_recovery_tcp_fallback_admissions` +counts TCP admissions per connection. -GetUrmaRecoveryStatus() and /vars/urma_recovery_status expose failure reasons, -cooldown state and negotiation counts. /vars/urma_recovery_tcp_fallback_admissions -counts TCP admissions. Mode changes apply to new connections. Existing connections -keep their negotiated transport; URMA admission resumes after cooldown. +### Functional checks -`example/urma_performance` builds `urma_recovery_client` from the shared recovery -client source using the URMA example dependencies, including URMA-only builds. -Select URMA with `--transport=urma` and use modes and CSV counters for two-host checks: +Use the `urma_performance_server` and `urma_recovery_client` built above with the +same path on both peers. Client example: ```bash -cmake --build build/urma-example --target urma_performance_server urma_recovery_client -j8 ./build/urma-example/urma_recovery_client --transport=urma \ - --server=SERVER_IP:8003 --urma_device=URMA_DEVICE \ + --server=SERVER_IP:8003 --urma_device=URMA_DEVICE --urma_tp_type=rtp \ --mode=auto --connection_type=short --seconds=15 --interval_ms=200 ``` -Configure `build/urma-example` as described above, linking the current URMA brpc -static library. Run `auto` and `tcp_only` separately and check RPC success, -URMA negotiation and TCP admission counters. Use `--mode_file` to switch between -`auto` and `tcp_only` while running; short connections exercise new admission. -Exit code 0 means at least one RPC succeeded; a healthy baseline also requires -`failure=0`. Functional checks cover mode changes, fault counters and recovery; -throughput and P99 belong to a separate performance test. +Healthy AUTO requires `failure=0`, increasing URMA negotiations and zero TCP +admissions. TCP_ONLY requires `failure=0` and increasing TCP admissions. +Use `--mode_file` to switch modes while running. ### Targeted send failure test -The URMA example provides an optional Linux `LD_PRELOAD` module that rejects one -data SEND with EIO in the selected client process. Other calls, including credit -ACKs, are forwarded to liburma. +Enable the Linux injector to reject one data SEND with EIO in the selected client: ```bash cmake -S example/urma_performance -B build/urma-example -DBUILD_URMA_FAULT_INJECTOR=ON @@ -289,21 +251,16 @@ python3 example/urma_performance/run_recovery_injection.py \ --server=SERVER_IP:8003 --device=URMA_DEVICE --tp-type=rtp ``` -The runner waits for successful URMA RPCs before creating a unique marker. It -queries the client's loopback metrics directly and checks the listener PID. -CSV, logs and baseline/cooldown/recovered snapshots are saved in a new directory. -Checks require exactly one failed RPC, `failures.send_post=1`, TCP admission during -cooldown and resumed URMA negotiations. Defaults are v3, a 40-second run and -10-second cooldown, with a separate startup timeout. This tests SEND_POST handling -and recovery policy; device completion errors require separate injection. Build -`urma_send_failure_injector_test` and run CTest in the example build directory -for the tool self-tests. - -## Comparing send access failures - -With `--urma_copy_send=true`, each data WR uses a fresh, 64-byte aligned registered -buffer held by its send slot until completion. Message and attachment contents are -preserved. This mode adds a CPU copy. Compare it with the original send path while -keeping device, queue and payload settings the same on both peers. Startup logs -include `copy_send`; failure logs include direction, send-slot addresses, lengths -and registered segment ranges. +The runner waits for successful URMA RPCs, checks the metric listener PID, and +saves CSV, logs and three phase snapshots. Defaults are v3, a 40-second run and +10-second cooldown. Checks require one expected failure, `send_post=1`, increasing +TCP admissions during cooldown and resumed URMA negotiations afterward. +Build `urma_send_failure_injector_test` and run CTest in the example build +directory for the tool self-tests. + +### Comparing send access failures + +`--urma_copy_send=true` coalesces each data WR into a 64-byte aligned registered +buffer held until completion, adding one CPU copy. Keep device, queue and payload +settings consistent when comparing send paths. Startup logs report `copy_send`; +failure logs report direction, send slot and registered segment range. diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md index ed6c22f73d..cf3a18497e 100644 --- a/example/rdma_performance/RECOVERY.md +++ b/example/rdma_performance/RECOVERY.md @@ -19,8 +19,8 @@ under the License. # RDMA 双机故障恢复测试 -两台 Linux 主机运行 RDMA 服务端和 recovery_client,TCP 地址通过独立可达的网络通信, -故障注入作用于 RDMA 测试链路。记录设备、端口、GID、驱动和固件版本。 +两台 Linux 主机运行服务端和 recovery_client,TCP 控制地址保持可达。 +记录设备、端口、GID、驱动和固件版本。 ## 编译 @@ -34,17 +34,15 @@ cmake -S example/rdma_performance -B build/recovery -DCMAKE_PREFIX_PATH="$PWD/bu cmake --build build/recovery --target server recovery_client -j8 ``` -Bazel 入口为 `//example:rdma_recovery_client`,Make 入口为示例目录中的 `recovery_client`。 - ## 运行与切换 -主机 B 启动服务端,按环境选择 RDMA 设备及 GID: +服务端: ```bash ./build/recovery/server --port=8002 --use_rdma=true --rdma_device=mlx5_0 --rdma_gid_index=3 ``` -主机 A 启动客户端,将 B_TCP_IP 替换为服务端 TCP 地址: +客户端,将 B_TCP_IP 替换为服务端地址: ```bash printf 'auto\n' > /tmp/brpc-rdma-mode @@ -53,50 +51,34 @@ printf 'auto\n' > /tmp/brpc-rdma-mode --rdma_device=mlx5_0 --rdma_gid_index=3 ``` -客户端持续发送 RPC,失败后继续下一次调用。每秒输出累计成功数、失败数、RDMA 协商数、 -TCP 回落准入数、冷却对端数和端口状态。运行中修改模式文件: +在客户端另一终端切换模式并查看指标: ```bash printf 'tcp_only\n' > /tmp/brpc-rdma-mode +curl --noproxy '*' 'http://127.0.0.1:8001/vars/rdma_recovery_*' printf 'auto\n' > /tmp/brpc-rdma-mode ``` -`short` 为每次调用建立连接,便于验证准入和冷却后的 RDMA 重建。 -`single` 用于观察存量连接:已回落的 TCP 连接继续复用,模式切换影响后续新连接。 - -客户端诊断入口: +`short` 每次调用建立连接,`single` 复用已协商的传输。 +CSV 每秒输出累计成功、失败、传输协商、TCP 准入及冷却状态。 +健康阶段检查 `failure=0`;注入阶段记录新增失败和恢复时延。 +`rdma_recovery_status` 提供分原因故障计数,TCP 准入指标按连接累计。 -```bash -curl 'http://127.0.0.1:8001/vars/rdma_recovery_*' -``` +## 验收 -`rdma_recovery_status` 提供模式、epoch、端口状态、冷却剩余时间和分原因故障计数; -`rdma_recovery_tcp_fallback_admissions` 为策略选择 TCP 的累计准入次数。 -计数以连接准入为单位,独立于 RPC 成功数。 - -## 测试步骤 - -| 场景 | 操作 | 观测 | +| 场景 | 操作 | 预期 | | --- | --- | --- | -| 健康链路 | AUTO 运行 | RPC 成功,RDMA 协商数持续增加 | -| 运维降级 | 文件改为 tcp_only | 后续连接成功,TCP 准入数增加,RDMA 协商数停止增加 | -| 恢复模式 | 文件改回 auto | short 模式下 RDMA 协商数恢复增加 | -| 端口故障 | 关闭客户端 RDMA 测试端口,保持 TCP 地址可达 | port_active=false,新连接经 TCP 成功 | -| 端口恢复 | 恢复测试端口,等待 cooldown_ms | 端口变为 active,冷却结束后 RDMA 协商数恢复增加 | -| 在途故障 | single 模式下断开 RDMA 链路 | 记录在途失败和恢复时延,后续连接经 TCP 继续调用 | -| 对端故障 | 中断服务端 RDMA 数据路径,保持 TCP 服务 | retry_exhausted / qp_fatal 等计数增加,对端进入冷却 | - -RNR 测试需要接收端故障注入,暂停补充接收 WR 并突破接收 credit 保护后观察 rnr 计数。 -普通拥塞可能只表现为背压;源码 mock CQE 测试可确定性验证 RNR 分支。 +| 正常通信 | AUTO 运行 | RPC 成功,RDMA 协商数增加 | +| 运行中降级 | 切换 TCP_ONLY | TCP 准入增加,RDMA 协商暂停 | +| 恢复 RDMA | 切回 AUTO | short 连接的 RDMA 协商恢复 | +| 端口故障 | 关闭 RDMA 测试端口 | port_active=false,后续连接使用 TCP | +| 端口恢复 | 恢复端口并等待冷却 | RDMA 准入恢复 | +| 对端故障 | 中断服务端 RDMA 数据路径 | 故障计数增加,对端冷却,后续连接使用 TCP | +| RNR | 暂停接收 WR 补充并突破 credit 保护 | rnr 计数增加,对端冷却 | -使用 `--use_rdma=false` 与 TCP 服务端可做无网卡烟测。 -客户端退出码 0 表示至少一次 RPC 成功,2 表示整个测试均失败;故障窗口内的失败数单独查看。 -验收记录故障前后成功率、恢复时延和各原因计数;性能基线使用原 performance client。 +使用 `single` 观察在途失败,使用 `short` 检查新连接的回落和恢复。 ## URMA -链接启用 URMA 的 brpc,服务端配置 `ServerOptions.socket_mode = SOCKET_MODE_URMA`, -客户端增加 `--transport=urma`。`--mode`、`--mode_file`、`--cooldown_ms` 和连接类型沿用上述配置。 -CSV 的 `transport_negotiated` 记录选定传输的协商数,诊断入口使用 `/vars/urma_recovery_*`。 -断开 URMA 数据路径并保持 TCP 可达,观察重试耗尽或 RNR 计数、对端冷却和后续 TCP 调用; -冷却到期后使用 short 连接验证 URMA 重新准入。 +[URMA 测试入口](../../docs/cn/urma.md#运行期故障恢复)提供恢复客户端及 SEND 失败注入运行器, +支持 v2/v3 握手、模式切换、对端冷却、TCP 准入和 URMA 恢复。 From 2e10c38439767e953560f0e7d96ceca7c8993234 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:21:59 -0700 Subject: [PATCH 19/29] Add attachment validation and persistent connection recovery checks --- docs/cn/urma.md | 8 ++- docs/en/urma.md | 10 +++- example/rdma_performance/recovery_client.cpp | 27 ++++++++-- .../run_recovery_injection.py | 51 ++++++++++++++----- .../test_run_recovery_injection.py | 7 +++ test/brpc_urma_transport_unittest.cpp | 3 +- 6 files changed, 85 insertions(+), 21 deletions(-) diff --git a/docs/cn/urma.md b/docs/cn/urma.md index c7df2ad228..48c822c622 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -223,6 +223,8 @@ URMA 与 RDMA 独立维护模式、epoch 和冷却缓存,模式切换过滤旧 健康 AUTO 检查 `failure=0`、URMA 协商增加、TCP 准入为 0; TCP_ONLY 检查 `failure=0`、TCP 准入增加。运行中通过 `--mode_file` 切换模式。 +使用 `--connection_type=single` 或 `pooled` 验证连接复用,`--attachment_size=1048576` +验证 1 MiB 回显附件;客户端逐字节校验,汇总输出 `validation_failures`。 ### 定向发送失败测试 @@ -236,8 +238,10 @@ python3 example/urma_performance/run_recovery_injection.py \ ``` 运行器等待 URMA RPC 成功后注入,校验指标 PID,保存 CSV、日志和三阶段快照。 -默认 v3 握手、运行 40 秒、冷却 10 秒;验收一次预期失败、`send_post=1`、 -冷却期间 TCP 准入增加、冷却后 URMA 协商恢复。 +默认 v3 握手、运行 40 秒、冷却 10 秒;验收 `send_post=1` 和冷却期间 TCP 准入。 +`short` 检查一次预期 RPC 失败和冷却后 URMA 协商恢复;`--connection-type=single` 或 +`pooled` 检查重连后 RPC 持续成功,复用已建立的 TCP 连接。 +运行器支持 `--attachment-size` 回显校验和 `--polling` 轮询模式。 构建 `urma_send_failure_injector_test` 后,通过示例目录的 CTest 执行工具自测。 ### 发送访问错误对照 diff --git a/docs/en/urma.md b/docs/en/urma.md index 5857b272ec..03cfc19098 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -239,6 +239,9 @@ same path on both peers. Client example: Healthy AUTO requires `failure=0`, increasing URMA negotiations and zero TCP admissions. TCP_ONLY requires `failure=0` and increasing TCP admissions. Use `--mode_file` to switch modes while running. +Use `--connection_type=single` or `pooled` for connection reuse and +`--attachment_size=1048576` for a 1 MiB echo attachment. The client checks every +byte and reports `validation_failures` in the summary. ### Targeted send failure test @@ -253,8 +256,11 @@ python3 example/urma_performance/run_recovery_injection.py \ The runner waits for successful URMA RPCs, checks the metric listener PID, and saves CSV, logs and three phase snapshots. Defaults are v3, a 40-second run and -10-second cooldown. Checks require one expected failure, `send_post=1`, increasing -TCP admissions during cooldown and resumed URMA negotiations afterward. +10-second cooldown. Checks require `send_post=1` and TCP admission during cooldown. +`short` requires one expected RPC failure and resumed URMA negotiations; +`--connection-type=single` or `pooled` checks sustained RPC recovery after +reconnecting and reusing TCP. The runner supports `--attachment-size` for echo +validation and `--polling` for polling mode. Build `urma_send_failure_injector_test` and run CTest in the example build directory for the tool self-tests. diff --git a/example/rdma_performance/recovery_client.cpp b/example/rdma_performance/recovery_client.cpp index 171a64f500..c0673620b7 100644 --- a/example/rdma_performance/recovery_client.cpp +++ b/example/rdma_performance/recovery_client.cpp @@ -39,6 +39,7 @@ DEFINE_int32(interval_ms, 100, "Pause between RPCs"); DEFINE_int32(timeout_ms, 1000, "RPC deadline"); DEFINE_int32(cooldown_ms, 5000, "Path cooldown"); DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); +DEFINE_int32(attachment_size, 0, "Echo attachment bytes and verify the returned content"); static bool SetMode(const std::string& value) { using brpc::rdma::RdmaRecoveryMode; @@ -70,9 +71,10 @@ int main(int argc, char* argv[]) { brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms) : brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms); if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || + FLAGS_attachment_size < 0 || !cooldown_ok || !SetMode(FLAGS_mode)) { - std::cerr << "Invalid duration, interval, timeout, cooldown or mode\n"; + std::cerr << "Invalid duration, interval, timeout, cooldown, mode or attachment size\n"; return 1; } brpc::Channel channel; @@ -92,6 +94,11 @@ int main(int argc, char* argv[]) { std::string mode = FLAGS_mode; uint64_t succeeded = 0; uint64_t failed = 0; + uint64_t validation_failed = 0; + std::string payload(static_cast(FLAGS_attachment_size), '\0'); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i % 251); + } const int64_t start = butil::monotonic_time_us(); int64_t next_report = start; std::cout << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," @@ -110,13 +117,23 @@ int main(int argc, char* argv[]) { } } test::PerfTestRequest request; - request.set_echo_attachment(false); + request.set_echo_attachment(!payload.empty()); test::PerfTestResponse response; brpc::Controller cntl; + const uint64_t sequence = succeeded + failed; + for (size_t i = 0; i < payload.size() && i < sizeof(sequence); ++i) { + payload[i] = static_cast(sequence >> (i * 8)); + } + cntl.request_attachment().append(payload); test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, nullptr); if (cntl.Failed()) { ++failed; std::cerr << "RPC failure: " << cntl.ErrorText() << '\n'; + } else if (!payload.empty() && cntl.response_attachment().to_string() != payload) { + ++failed; + ++validation_failed; + std::cerr << "Attachment mismatch: expected=" << payload.size() + << " actual=" << cntl.response_attachment().size() << '\n'; } else { ++succeeded; } @@ -134,6 +151,10 @@ int main(int argc, char* argv[]) { } usleep(static_cast(FLAGS_interval_ms) * 1000); } - std::cout << "completed success=" << succeeded << " failure=" << failed << std::endl; + std::cout << "completed success=" << succeeded << " failure=" << failed; + if (!payload.empty()) { + std::cout << " validation_failures=" << validation_failed; + } + std::cout << std::endl; return succeeded ? 0 : 2; } diff --git a/example/urma_performance/run_recovery_injection.py b/example/urma_performance/run_recovery_injection.py index e16b7db1d4..4e172c80ff 100644 --- a/example/urma_performance/run_recovery_injection.py +++ b/example/urma_performance/run_recovery_injection.py @@ -60,7 +60,7 @@ def wait_for(proc, predicate, timeout, description): raise RuntimeError("Timed out waiting for " + description) -def validate_phases(baseline, cooldown, recovered): +def validate_phases(baseline, cooldown, recovered, require_urma_reconnect=True): if any(s["mode"] != "auto" for s in (baseline, cooldown, recovered)): raise RuntimeError("Expected AUTO throughout the run") if len({s["epoch"] for s in (baseline, cooldown, recovered)}) != 1: @@ -71,8 +71,10 @@ def validate_phases(baseline, cooldown, recovered): if (cooldown["failures"]["send_post"] != 1 or cooldown["cooled_peers"] != 1 or cooldown["peer_cooldown_admissions"] <= 0): raise RuntimeError("Expected one SEND failure and TCP peer-cooldown admission") + negotiated = recovered["negotiated_connections"] + previous = cooldown["negotiated_connections"] if (recovered["failures"]["send_post"] != 1 or recovered["cooled_peers"] != 0 - or recovered["negotiated_connections"] <= cooldown["negotiated_connections"] + or negotiated < previous or (require_urma_reconnect and negotiated == previous) or recovered["peer_cooldown_admissions"] < cooldown["peer_cooldown_admissions"]): raise RuntimeError("Expected URMA negotiations to resume after cooldown") @@ -112,7 +114,9 @@ def record_phase(name, status): "--server=" + args.server, "--urma_device=" + args.device, "--urma_tp_type=" + args.tp_type, "--urma_client_handshake_version=" + str(args.handshake_version), - "--urma_buffer_count=4096", "--connection_type=short", + "--urma_buffer_count=4096", "--connection_type=" + args.connection_type, + "--attachment_size=" + str(args.attachment_size), + "--urma_use_polling=" + str(args.polling).lower(), "--interval_ms=200", "--timeout_ms=3000", "--seconds=" + str(args.seconds), "--cooldown_ms=" + str(args.cooldown_ms), "--dummy_port=" + str(port)] @@ -149,30 +153,47 @@ def cooling(): and status["peer_cooldown_admissions"] > 0) else None phases["cooldown"] = record_phase("cooldown", wait_for( - proc, cooling, 5, "peer cooldown and TCP admission")) + proc, cooling, args.cooldown_ms / 1000, "peer cooldown and TCP admission")) + require_urma_reconnect = args.connection_type == "short" + recovery_success = None def recovering(): + nonlocal recovery_success status = read_var("urma_recovery_status") - return status if (status["cooled_peers"] == 0 and - status["negotiated_connections"] > - phases["cooldown"]["negotiated_connections"]) else None + if status["cooled_peers"] != 0: + return None + if require_urma_reconnect: + return status if (status["negotiated_connections"] > + phases["cooldown"]["negotiated_connections"]) else None + rows = read_samples(csv_path) + if recovery_success is None: + recovery_success = rows[-1][1] + return status if rows[-1][1] > recovery_success else None phases["recovered"] = record_phase("recovered", wait_for( proc, recovering, args.cooldown_ms / 1000 + 5, "URMA recovery")) - validate_phases(**phases) + validate_phases(**phases, require_urma_reconnect=require_urma_reconnect) + failures_at_recovery = read_samples(csv_path)[-1][2] (output / "snapshots.json").write_text(json.dumps( {"pid": proc.pid, "port": port, "phases": phases}, indent=2)) rc = proc.wait(timeout=args.seconds + 10) summary = re.search(r"completed success=(\d+) failure=(\d+)", csv_path.read_text()) - if rc != 0 or not summary or int(summary[1]) == 0 or int(summary[2]) != 1: - raise RuntimeError("Expected exactly one failed RPC and remaining RPCs to succeed") + expected_failures = 1 if require_urma_reconnect else failures_at_recovery + if (rc != 0 or not summary or int(summary[1]) == 0 + or expected_failures < 1 or int(summary[2]) != expected_failures): + raise RuntimeError("Expected RPC recovery with no further failures afterward") + if args.attachment_size: + validation = re.search(r"validation_failures=(\d+)", csv_path.read_text()) + if not validation or int(validation[1]) != 0: + raise RuntimeError("Attachment content validation failed") if log_path.read_text().count("injected one data SEND post failure") != 1: raise RuntimeError("Expected exactly one injection log") last = read_samples(csv_path)[-1] if last[5] != 0 or last[4] != phases["recovered"]["peer_cooldown_admissions"]: raise RuntimeError("TCP cooldown admissions must stop after URMA recovery") - print(summary[0]) - print("PASS: one SEND failure, TCP cooldown, and URMA recovery") + print(csv_path.read_text().splitlines()[-1]) + recovered_transport = "URMA" if require_urma_reconnect else "RPC" + print("PASS: one SEND failure, TCP cooldown, and " + recovered_transport + " recovery") finally: if proc is not None and proc.poll() is None: proc.terminate() @@ -191,12 +212,16 @@ def main(): parser.add_argument("--injector", default="build/urma-example/liburma_send_failure_injector.so") parser.add_argument("--tp-type", choices=("rtp", "ctp", "auto"), default="rtp") parser.add_argument("--handshake-version", type=int, choices=(2, 3), default=3) + parser.add_argument("--connection-type", choices=("short", "single", "pooled"), default="short") + parser.add_argument("--attachment-size", type=int, default=0) + parser.add_argument("--polling", action="store_true") parser.add_argument("--seconds", type=int, default=40) parser.add_argument("--cooldown-ms", type=int, default=10000) parser.add_argument("--baseline-seconds", type=float, default=5) parser.add_argument("--startup-timeout", type=float, default=120) args = parser.parse_args() - if (args.cooldown_ms <= 0 or args.baseline_seconds < 0 or args.startup_timeout <= 0 + if (args.cooldown_ms <= 0 or args.attachment_size < 0 + or args.baseline_seconds < 0 or args.startup_timeout <= 0 or args.seconds < args.baseline_seconds + args.cooldown_ms / 1000 + 5): parser.error("Duration must cover baseline, cooldown and at least five seconds of recovery") try: diff --git a/example/urma_performance/test_run_recovery_injection.py b/example/urma_performance/test_run_recovery_injection.py index a4991857cb..e1d42b3d83 100644 --- a/example/urma_performance/test_run_recovery_injection.py +++ b/example/urma_performance/test_run_recovery_injection.py @@ -59,6 +59,13 @@ def phases(self): def test_validate_complete_recovery(self): runner.validate_phases(*self.phases()) + def test_persistent_connections_can_reuse_tcp_after_cooldown(self): + baseline, cooldown, recovered = self.phases() + recovered["negotiated_connections"] = cooldown["negotiated_connections"] + runner.validate_phases(baseline, cooldown, recovered, require_urma_reconnect=False) + with self.assertRaises(RuntimeError): + runner.validate_phases(baseline, cooldown, recovered) + def test_reject_stale_zero_failure_snapshot(self): baseline, cooldown, recovered = self.phases() cooldown["failures"]["send_post"] = 0 diff --git a/test/brpc_urma_transport_unittest.cpp b/test/brpc_urma_transport_unittest.cpp index 94799b372e..f7ca8e87b8 100644 --- a/test/brpc_urma_transport_unittest.cpp +++ b/test/brpc_urma_transport_unittest.cpp @@ -116,7 +116,8 @@ TEST_P(UrmaTransportTest, exchanges_zero_copy_payloads_and_receive_credits) { const auto before = brpc::urma::GetUrmaRecoveryStatus(); ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); for (int round = 0; round < 40; ++round) { - const std::string payload(round == 0 ? 0 : 20000, 'r'); + const size_t size = round == 0 ? 0 : round == 1 ? 1024 * 1024 : 20000; + const std::string payload(size, 'r'); brpc::Controller cntl; test::EchoRequest request; test::EchoResponse response; From 7af9741130f21792d8d6b3a44ff802d57139a140 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:33:24 -0700 Subject: [PATCH 20/29] Move recovery coverage into independent unit test files --- test/brpc_rdma_runtime_unittest.cpp | 510 +++++++++++++++++++++++ test/brpc_rdma_unittest.cpp | 270 ------------ test/brpc_urma_runtime_unittest.cpp | 622 ++++++++++++++++++++++++++++ test/brpc_urma_unittest.cpp | 525 ----------------------- 4 files changed, 1132 insertions(+), 795 deletions(-) create mode 100644 test/brpc_rdma_runtime_unittest.cpp create mode 100644 test/brpc_urma_runtime_unittest.cpp diff --git a/test/brpc_rdma_runtime_unittest.cpp b/test/brpc_rdma_runtime_unittest.cpp new file mode 100644 index 0000000000..a1db214a5a --- /dev/null +++ b/test/brpc_rdma_runtime_unittest.cpp @@ -0,0 +1,510 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include +#include +#include +#if BRPC_WITH_RDMA +#include +#include +#include +#include +#include +#include "butil/endpoint.h" +#include "butil/fd_guard.h" +#include "butil/iobuf.h" +#include "butil/sys_byteorder.h" +#include "butil/time.h" +#include "brpc/acceptor.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/input_messenger.h" +#include "brpc/server.h" +#include "brpc/socket.h" +#include "brpc/errno.pb.h" +#include "brpc/rdma_transport.h" +#include "brpc/rdma/rdma_endpoint.h" +#include "brpc/rdma/rdma_handshake.h" +#include "brpc/rdma/rdma_handshake_constants.h" +#include "brpc/rdma/rdma_handshake.pb.h" +#include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" +#include "echo.pb.h" + +using namespace brpc; + +namespace brpc { +namespace rdma { +extern butil::atomic g_rdma_available; +extern bool g_skip_rdma_init; +} // namespace rdma +} // namespace brpc + +static butil::EndPoint g_ep; +static const int64_t WAIT_TIMEOUT_US = 5000000; +static const int64_t WAIT_INTERVAL_US = 1000; + +static bool WaitUntil(const std::function& pred, + int64_t timeout_us = WAIT_TIMEOUT_US) { + const int64_t deadline = butil::gettimeofday_us() + timeout_us; + while (!pred()) { + if (butil::gettimeofday_us() >= deadline) { + return false; + } + usleep((useconds_t)WAIT_INTERVAL_US); + } + return true; +} + +// write(2) may transfer less than asked for, so a test that cares about the +// whole buffer reaching the peer has to loop. +static bool WriteAll(int fd, const void* buf, size_t len) { + const uint8_t* p = (const uint8_t*)buf; + for (size_t done = 0; done < len; ) { + ssize_t n = write(fd, p + done, len - done); + if (n < 0) { + if (errno == EINTR) { + continue; + } + return false; + } + done += n; + } + return true; +} + +static void ConnectToServer(butil::fd_guard* sockfd) { + sockaddr_in addr; + bzero((char*)&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(g_ep.port); + addr.sin_addr = g_ep.ip; + sockfd->reset(socket(AF_INET, SOCK_STREAM, 0)); + ASSERT_TRUE(*sockfd >= 0); + timeval tv; + tv.tv_sec = WAIT_TIMEOUT_US / 1000000; + tv.tv_usec = WAIT_TIMEOUT_US % 1000000; + ASSERT_EQ(0, setsockopt(*sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))); + ASSERT_EQ(0, connect(*sockfd, (sockaddr*)&addr, sizeof(addr))); +} + +class RecoveryEchoService : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, const ::test::EchoRequest*, + ::test::EchoResponse* response, + google::protobuf::Closure* done) override { + ClosureGuard done_guard(done); + response->set_message("MyEchoService"); + } +}; + +class RdmaRuntimeFixture : public ::testing::Test { +protected: + RdmaRuntimeFixture() { + EXPECT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &g_ep)); + EXPECT_EQ(0, _server.AddService(&_service, SERVER_DOESNT_OWN_SERVICE)); + } + + void StartServer(bool use_rdma = true) { + ServerOptions options; + options.enabled_protocols = "baidu_std"; + options.socket_mode = use_rdma ? SOCKET_MODE_RDMA : SOCKET_MODE_TCP; + options.idle_timeout_sec = 5; + ASSERT_EQ(0, _server.Start("127.0.0.1:0", &options)); + g_ep = _server.listen_address(); + } + + void StopServer() { + _server.Stop(0); + _server.Join(); + } + + Socket* GetSocketFromServer(size_t index) { + std::vector sids; + _server._am->ListConnections(&sids); + if (index >= sids.size()) { + return nullptr; + } + SocketUniquePtr socket; + return Socket::Address(sids[index], &socket) == 0 ? socket.get() : nullptr; + } + + Socket* WaitForServerSocket() { + Socket* socket = nullptr; + WaitUntil([this, &socket] { + return (socket = GetSocketFromServer(0)) != nullptr; + }); + return socket; + } + + bool WaitForServerSocketGone() { + return WaitUntil([this] { return GetSocketFromServer(0) == nullptr; }); + } + + Server _server; + RecoveryEchoService _service; +}; + +static RdmaTransport* RdmaTransportOf(Socket* s) { + return static_cast(s->_transport.get()); +} +static RdmaTransport* RdmaTransportOf(const SocketUniquePtr& s) { + return RdmaTransportOf(s.get()); +} + +// Polls until the endpoint reaches `expected` and returns the last state seen, +// so that ASSERT_RDMA_STATE() reports what the endpoint actually settled on. +static rdma::RdmaEndpoint::State WaitForRdmaState( + RdmaTransport* transport, rdma::RdmaEndpoint::State expected) { + rdma::RdmaEndpoint::State state = transport->_rdma_ep->_state; + WaitUntil([transport, expected, &state] { + state = transport->_rdma_ep->_state; + return state == expected; + }); + return state; +} + +// Waits for `transport` to reach `expected`, failing the test if it does not. +#define ASSERT_RDMA_STATE(expected, transport) \ + ASSERT_EQ(expected, WaitForRdmaState(transport, expected)) + +static void MakeV2ClientHello(uint8_t (&data)[rdma::HELLO_V2_MSG_LEN_MIN]) { + rdma::v2_wire::HelloMessage msg{}; + msg.msg_len = rdma::HELLO_V2_MSG_LEN_MIN; + msg.hello_ver = rdma::HELLO_V2_VERSION; + msg.impl_ver = rdma::IMPL_V2_VERSION; + msg.sq_size = 16; + msg.rq_size = 16; + msg.block_size = 8192; + msg.qp_num = 0; + msg.gid = rdma::GetRdmaGid(); + memcpy(data, "RDMA", 4); + msg.Serialize(data + 4); +} + +std::string MakeV3Packet(const rdma::RdmaHello& msg) { + std::string body; + EXPECT_TRUE(msg.SerializeToString(&body)); + std::string packet; + packet.reserve(4 + 4 + body.size()); + packet.append("RDM3", 4); + uint32_t pb_size_be = + butil::HostToNet32(static_cast(body.size())); + packet.append(reinterpret_cast(&pb_size_be), 4); + packet.append(body); + return packet; +} + +// Build a fully-valid RdmaHello: all 6 required fields are set, with +// values that pass RdmaHelloV3Wire::RdmaHelloValid(). +// - block_size = 8192 (>= MIN_BLOCK_SIZE) +// - sq_size / rq_size = 16 (>= MIN_QP_SIZE) +// - gid = exactly 16B (sizeof(ibv_gid)) +// - qp_num = 0 (allowed because g_skip_rdma_init in UT) +rdma::RdmaHello MakeValidV3Hello() { + rdma::RdmaHello msg; + msg.set_block_size(8192); + msg.set_sq_size(16); + msg.set_rq_size(16); + msg.set_lid(0); + ibv_gid gid = rdma::GetRdmaGid(); + msg.set_gid(std::string(reinterpret_cast(gid.raw), + sizeof(gid.raw))); + msg.set_qp_num(0); + return msg; +} + + +class RdmaAdmissionTest : public RdmaRuntimeFixture { +protected: + void SetUp() override { + _saved_mode = rdma::GetRdmaRecoveryMode(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + } + + void TearDown() override { + StopServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(_saved_mode); + } + + void CallEchoOverFallback() { + Channel channel; + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.connect_timeout_ms = 1000; + options.timeout_ms = 1000; + options.max_retry = 0; + ASSERT_EQ(0, channel.Init(g_ep, &options)); + Controller cntl; + test::EchoRequest req; + test::EchoResponse res; + req.set_message("runtime recovery"); + test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ("MyEchoService", res.message()); + SocketUniquePtr s; + ASSERT_EQ(0, Socket::Address(cntl._single_server_id, &s)); + EXPECT_EQ(RdmaTransport::RDMA_OFF, RdmaTransportOf(s)->_rdma_state); + EXPECT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, + RdmaTransportOf(s)->_rdma_ep->_state); + EXPECT_FALSE(s->Failed()); + } + +private: + rdma::RdmaRecoveryMode _saved_mode; +}; + +TEST_F(RdmaAdmissionTest, tcp_only_carries_rpc_to_tcp_server) { + StartServer(false); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, cooled_peer_carries_rpc_over_tcp) { + StartServer(); + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + policy.ReportPeerFailure(peer, epoch); + ASSERT_FALSE(policy.AllowNewConnection(peer)); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +static int PollRnrCompletion(ibv_cq* cq, int, ibv_wc* wc) { + bool* delivered = static_cast(cq->cq_context); + if (*delivered) { + return 0; + } + *delivered = true; + memset(wc, 0, sizeof(*wc)); + wc->status = IBV_WC_RNR_RETRY_EXC_ERR; + return 1; +} + +TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { + StartServer(); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + SocketOptions cq_options; + cq_options.user = ep; + SocketId cq_sid; + ASSERT_EQ(0, Socket::Create(cq_options, &cq_sid)); + SocketUniquePtr cq_socket; + ASSERT_EQ(0, Socket::Address(cq_sid, &cq_socket)); + + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + bool delivered = false; + ibv_context context{}; + context.ops.poll_cq = PollRnrCompletion; + ibv_cq cq{}; + cq.context = &context; + cq.cq_context = &delivered; + rdma::RdmaResource resource; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->_cq_sid = cq_sid; + ep->_recovery_epoch = epoch; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const bool saved_polling = rdma::FLAGS_rdma_use_polling; + rdma::FLAGS_rdma_use_polling = true; + + rdma::RdmaEndpoint::PollCq(cq_socket.get()); + const auto first_report = rdma::GetRdmaRecoveryStatus(); + ep->ReportFailure(rdma::RdmaFailureReason::QP_FATAL); + EXPECT_EQ(first_report.failures[static_cast(rdma::RdmaFailureReason::QP_FATAL)], + rdma::GetRdmaRecoveryStatus().failures[ + static_cast(rdma::RdmaFailureReason::QP_FATAL)]); + EXPECT_TRUE(delivered); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); + + // The test owns the mock CQ and resource; ordinary recycling owns neither. + ep->_resource = nullptr; + ep->_cq_sid = INVALID_SOCKET_ID; + resource.polling_cq = nullptr; + cq_socket->_user = nullptr; + cq_socket->SetFailed(); + rdma::FLAGS_rdma_use_polling = saved_polling; + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, server_declines_v2_and_v3_when_tcp_only) { + StartServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + for (int version : {2, 3}) { + butil::fd_guard fd; + ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); + if (version == 2) { + uint8_t hello[rdma::HELLO_V2_MSG_LEN_MIN]; + MakeV2ClientHello(hello); + ASSERT_TRUE(WriteAll(fd, hello, sizeof(hello))); + } else { + const std::string hello = MakeV3Packet(MakeValidV3Hello()); + ASSERT_TRUE(WriteAll(fd, hello.data(), hello.size())); + } + Socket* s = WaitForServerSocket(); + ASSERT_TRUE(s != nullptr); + auto* transport = RdmaTransportOf(s); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::S_ACK_WAIT, transport); + EXPECT_EQ(RdmaTransport::RDMA_OFF, transport->_rdma_state); + EXPECT_EQ(nullptr, transport->_rdma_ep->_resource); + const uint32_t ack = 0; + ASSERT_TRUE(WriteAll(fd, &ack, sizeof(ack))); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::FALLBACK_TCP, transport); + EXPECT_FALSE(s->Failed()); + fd.reset(-1); + ASSERT_TRUE(WaitForServerSocketGone()); + } +} + +TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RETRY_EXC_ERR)); + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RNR_RETRY_EXC_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_SUCCESS)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_WR_FLUSH_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_LOC_PROT_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); +} + +TEST_F(RdmaAdmissionTest, async_errors_validate_owner_and_report_once) { + for (ibv_event_type type : {IBV_EVENT_QP_FATAL, IBV_EVENT_CQ_ERR, + IBV_EVENT_QP_ACCESS_ERR}) { + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_qp qp{}; + ibv_qp other_qp{}; + ibv_cq cq{}; + ibv_cq other_cq{}; + rdma::RdmaResource resource; + resource.qp = &qp; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->BindResourceToSocket(); + EXPECT_EQ(sid, reinterpret_cast(qp.qp_context)); + EXPECT_EQ(sid, reinterpret_cast(cq.cq_context)); + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const auto before = policy.GetStatus(); + ibv_async_event event{}; + event.event_type = type; + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &other_cq; + } else { + event.element.qp = &other_qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_FALSE(socket->Failed()); + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &cq; + } else { + event.element.qp = &qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_TRUE(socket->Failed()); + const auto reason = type == IBV_EVENT_CQ_ERR ? rdma::RdmaFailureReason::CQ_ERROR : + type == IBV_EVENT_QP_FATAL ? rdma::RdmaFailureReason::QP_FATAL : + rdma::RdmaFailureReason::ACCESS_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + policy.GetStatus().failures[static_cast(reason)]); + EXPECT_EQ(type == IBV_EVENT_QP_ACCESS_ERR, policy.AllowNewConnection(peer)); + ep->ReportFailure(rdma::RdmaFailureReason::RNR); + EXPECT_EQ(before.failures[static_cast(rdma::RdmaFailureReason::RNR)], + policy.GetStatus().failures[static_cast(rdma::RdmaFailureReason::RNR)]); + const std::string exposed = bvar::Variable::describe_exposed("rdma_recovery_status"); + EXPECT_NE(std::string::npos, exposed.find("failures")); + EXPECT_NE(std::string::npos, exposed.find("qp_fatal")); + ep->_resource = nullptr; + resource.qp = nullptr; + resource.polling_cq = nullptr; + } +} + +static int FailAckPost(ibv_qp*, ibv_send_wr*, ibv_send_wr**) { + return EIO; +} + +TEST_F(RdmaAdmissionTest, ack_post_error_propagates_and_cools_peer) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_context context{}; + context.ops.post_send = FailAckPost; + ibv_qp qp{}; + qp.context = &context; + rdma::RdmaResource resource; + resource.qp = &qp; + ep->_resource = &resource; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + ep->_remote_window_capacity = 0; + ep->_new_rq_wrs.store(1); + ibv_wc wc{}; + wc.opcode = IBV_WC_SEND; + EXPECT_EQ(-1, ep->HandleCompletion(wc)); + EXPECT_EQ(EIO, errno); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_EQ(rdma::RdmaFailureReason::SEND_POST, ep->_runtime_failure.load()); + ep->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); +} + +#endif // BRPC_WITH_RDMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); +#if BRPC_WITH_RDMA + rdma::g_rdma_available.store(true, butil::memory_order_relaxed); + rdma::g_skip_rdma_init = true; +#endif + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index f6fe8eeff1..6ec54ed90b 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -35,7 +35,6 @@ #include "brpc/acceptor.h" #include "brpc/channel.h" #include "brpc/controller.h" -#include "brpc/input_messenger.h" #include "brpc/server.h" #include "brpc/socket.h" #include "brpc/errno.pb.h" @@ -48,7 +47,6 @@ #include "brpc/rdma/rdma_handshake_constants.h" #include "brpc/rdma/rdma_handshake.pb.h" #include "brpc/rdma/rdma_helper.h" -#include "brpc/rdma/rdma_recovery_policy.h" #include "echo.pb.h" static const int PORT = 8713; @@ -3318,274 +3316,6 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } -class RdmaAdmissionTest : public RdmaTest { -protected: - void SetUp() override { - _saved_mode = rdma::GetRdmaRecoveryMode(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); - } - - void TearDown() override { - StopServer(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(_saved_mode); - RdmaTest::TearDown(); - } - - void CallEchoOverFallback() { - Channel channel; - ChannelOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.connect_timeout_ms = 1000; - options.timeout_ms = 1000; - options.max_retry = 0; - ASSERT_EQ(0, channel.Init(g_ep, &options)); - Controller cntl; - test::EchoRequest req; - test::EchoResponse res; - req.set_message("runtime recovery"); - test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, nullptr); - ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); - EXPECT_EQ("MyEchoService", res.message()); - SocketUniquePtr s; - ASSERT_EQ(0, Socket::Address(cntl._single_server_id, &s)); - EXPECT_EQ(RdmaTransport::RDMA_OFF, RdmaTransportOf(s)->_rdma_state); - EXPECT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, - RdmaTransportOf(s)->_rdma_ep->_state); - EXPECT_FALSE(s->Failed()); - } - -private: - rdma::RdmaRecoveryMode _saved_mode; -}; - -TEST_F(RdmaAdmissionTest, tcp_only_carries_rpc_to_tcp_server) { - StartServer(false); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -TEST_F(RdmaAdmissionTest, cooled_peer_carries_rpc_over_tcp) { - StartServer(); - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - uint64_t epoch; - ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); - policy.ReportPeerFailure(peer, epoch); - ASSERT_FALSE(policy.AllowNewConnection(peer)); - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -static int PollRnrCompletion(ibv_cq* cq, int, ibv_wc* wc) { - bool* delivered = static_cast(cq->cq_context); - if (*delivered) { - return 0; - } - *delivered = true; - memset(wc, 0, sizeof(*wc)); - wc->status = IBV_WC_RNR_RETRY_EXC_ERR; - return 1; -} - -TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { - StartServer(); - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - SocketOptions cq_options; - cq_options.user = ep; - SocketId cq_sid; - ASSERT_EQ(0, Socket::Create(cq_options, &cq_sid)); - SocketUniquePtr cq_socket; - ASSERT_EQ(0, Socket::Address(cq_sid, &cq_socket)); - - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - uint64_t epoch; - ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); - bool delivered = false; - ibv_context context{}; - context.ops.poll_cq = PollRnrCompletion; - ibv_cq cq{}; - cq.context = &context; - cq.cq_context = &delivered; - rdma::RdmaResource resource; - resource.polling_cq = &cq; - ep->_resource = &resource; - ep->_cq_sid = cq_sid; - ep->_recovery_epoch = epoch; - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - const bool saved_polling = rdma::FLAGS_rdma_use_polling; - rdma::FLAGS_rdma_use_polling = true; - - rdma::RdmaEndpoint::PollCq(cq_socket.get()); - const auto first_report = rdma::GetRdmaRecoveryStatus(); - ep->ReportFailure(rdma::RdmaFailureReason::QP_FATAL); - EXPECT_EQ(first_report.failures[static_cast(rdma::RdmaFailureReason::QP_FATAL)], - rdma::GetRdmaRecoveryStatus().failures[ - static_cast(rdma::RdmaFailureReason::QP_FATAL)]); - EXPECT_TRUE(delivered); - EXPECT_TRUE(socket->Failed()); - EXPECT_FALSE(policy.AllowNewConnection(peer)); - EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); - - // The test owns the mock CQ and resource; ordinary recycling owns neither. - ep->_resource = nullptr; - ep->_cq_sid = INVALID_SOCKET_ID; - resource.polling_cq = nullptr; - cq_socket->_user = nullptr; - cq_socket->SetFailed(); - rdma::FLAGS_rdma_use_polling = saved_polling; - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -TEST_F(RdmaAdmissionTest, server_declines_v2_and_v3_when_tcp_only) { - StartServer(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); - for (int version : {2, 3}) { - butil::fd_guard fd; - ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); - if (version == 2) { - uint8_t hello[rdma::HELLO_V2_MSG_LEN_MIN]; - MakeV2ClientHello(hello); - ASSERT_TRUE(WriteAll(fd, hello, sizeof(hello))); - } else { - const std::string hello = MakeV3Packet(MakeValidV3Hello()); - ASSERT_TRUE(WriteAll(fd, hello.data(), hello.size())); - } - Socket* s = WaitForServerSocket(); - ASSERT_TRUE(s != nullptr); - auto* transport = RdmaTransportOf(s); - ASSERT_RDMA_STATE(rdma::RdmaEndpoint::S_ACK_WAIT, transport); - EXPECT_EQ(RdmaTransport::RDMA_OFF, transport->_rdma_state); - EXPECT_EQ(nullptr, transport->_rdma_ep->_resource); - const uint32_t ack = 0; - ASSERT_TRUE(WriteAll(fd, &ack, sizeof(ack))); - ASSERT_RDMA_STATE(rdma::RdmaEndpoint::FALLBACK_TCP, transport); - EXPECT_FALSE(s->Failed()); - fd.reset(-1); - ASSERT_TRUE(WaitForServerSocketGone()); - } -} - -TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { - EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RETRY_EXC_ERR)); - EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RNR_RETRY_EXC_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_SUCCESS)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_WR_FLUSH_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_LOC_PROT_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); -} - -TEST_F(RdmaAdmissionTest, async_errors_validate_owner_and_report_once) { - for (ibv_event_type type : {IBV_EVENT_QP_FATAL, IBV_EVENT_CQ_ERR, - IBV_EVENT_QP_ACCESS_ERR}) { - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); - ibv_qp qp{}; - ibv_qp other_qp{}; - ibv_cq cq{}; - ibv_cq other_cq{}; - rdma::RdmaResource resource; - resource.qp = &qp; - resource.polling_cq = &cq; - ep->_resource = &resource; - ep->BindResourceToSocket(); - EXPECT_EQ(sid, reinterpret_cast(qp.qp_context)); - EXPECT_EQ(sid, reinterpret_cast(cq.cq_context)); - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - const auto before = policy.GetStatus(); - ibv_async_event event{}; - event.event_type = type; - if (type == IBV_EVENT_CQ_ERR) { - event.element.cq = &other_cq; - } else { - event.element.qp = &other_qp; - } - rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); - EXPECT_FALSE(socket->Failed()); - if (type == IBV_EVENT_CQ_ERR) { - event.element.cq = &cq; - } else { - event.element.qp = &qp; - } - rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); - EXPECT_TRUE(socket->Failed()); - const auto reason = type == IBV_EVENT_CQ_ERR ? rdma::RdmaFailureReason::CQ_ERROR : - type == IBV_EVENT_QP_FATAL ? rdma::RdmaFailureReason::QP_FATAL : - rdma::RdmaFailureReason::ACCESS_ERROR; - EXPECT_EQ(before.failures[static_cast(reason)] + 1, - policy.GetStatus().failures[static_cast(reason)]); - EXPECT_EQ(type == IBV_EVENT_QP_ACCESS_ERR, policy.AllowNewConnection(peer)); - ep->ReportFailure(rdma::RdmaFailureReason::RNR); - EXPECT_EQ(before.failures[static_cast(rdma::RdmaFailureReason::RNR)], - policy.GetStatus().failures[static_cast(rdma::RdmaFailureReason::RNR)]); - const std::string exposed = bvar::Variable::describe_exposed("rdma_recovery_status"); - EXPECT_NE(std::string::npos, exposed.find("failures")); - EXPECT_NE(std::string::npos, exposed.find("qp_fatal")); - ep->_resource = nullptr; - resource.qp = nullptr; - resource.polling_cq = nullptr; - } -} - -static int FailAckPost(ibv_qp*, ibv_send_wr*, ibv_send_wr**) { - return EIO; -} - -TEST_F(RdmaAdmissionTest, ack_post_error_propagates_and_cools_peer) { - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); - ibv_context context{}; - context.ops.post_send = FailAckPost; - ibv_qp qp{}; - qp.context = &context; - rdma::RdmaResource resource; - resource.qp = &qp; - ep->_resource = &resource; - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - ep->_remote_window_capacity = 0; - ep->_new_rq_wrs.store(1); - ibv_wc wc{}; - wc.opcode = IBV_WC_SEND; - EXPECT_EQ(-1, ep->HandleCompletion(wc)); - EXPECT_EQ(EIO, errno); - EXPECT_FALSE(policy.AllowNewConnection(peer)); - EXPECT_EQ(rdma::RdmaFailureReason::SEND_POST, ep->_runtime_failure.load()); - ep->_resource = nullptr; - resource.qp = nullptr; - socket->SetFailed(); -} - #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { diff --git a/test/brpc_urma_runtime_unittest.cpp b/test/brpc_urma_runtime_unittest.cpp new file mode 100644 index 0000000000..29081ebb78 --- /dev/null +++ b/test/brpc_urma_runtime_unittest.cpp @@ -0,0 +1,622 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "butil/atomicops.h" +#include "butil/sys_byteorder.h" +#include "urma_api.h" +#include "brpc/urma/urma_handshake.h" +#include "brpc/urma/urma_handshake.pb.h" +#include "brpc/urma/urma_helper.h" +#include "urma_types.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/fd_guard.h" +#include "bvar/variable.h" +#include "echo.pb.h" + +using namespace brpc; + +namespace brpc { +namespace urma { + +DECLARE_int32(urma_client_handshake_version); +extern bool g_skip_urma_init; +extern butil::atomic g_urma_available; + +} // namespace urma +} // namespace brpc + +TEST(UrmaRuntimeHelperTest, combined_priority_and_invalid_transport_type) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); + EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( + attr, static_cast(255))); +} + +TEST(UrmaRuntimeHelperTest, auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); + attr.dev_cap.priority_info[7].tp_type.bs.ctp = 1; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_CTP, path); + EXPECT_EQ(7, priority); +} + +TEST(UrmaRuntimeHelperTest, explicit_path_uses_matching_priority_and_combined_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[2].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[2].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"rtp", "ctp"}) { + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + EXPECT_EQ(std::string(requested) == "rtp" ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(2, priority); + } +} + +TEST(UrmaRuntimeHelperTest, rejects_unsupported_paths_in_a_reported_capability_table) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "ctp", &path, &priority)); + EXPECT_EQ(URMA_UTP, path); + EXPECT_EQ(15, priority); + attr.dev_cap.priority_info[0].tp_type.value = 0; + attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); +} + +TEST(UrmaRuntimeHelperTest, providers_without_path_capabilities_keep_compatibility_priorities) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const auto& requested : {"auto", "ctp", "rtp"}) { + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + const bool rtp = std::string(requested) == "rtp"; + EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(rtp ? 0 : 6, priority); + } +} + +TEST(UrmaRuntimeHelperTest, rejects_invalid_path_configuration) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 6; + for (const auto& requested : {"", "utp", "RM", "typo"}) { + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + } + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, nullptr)); +} + +TEST(UrmaRuntimeHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge; + memset(&sge, 0xff, sizeof(sge)); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x1000, 8192, &segment)); + EXPECT_EQ(0x1000u, sge.addr); + EXPECT_EQ(8192u, sge.len); + EXPECT_EQ(&segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x2fff, 1, &segment)); + EXPECT_EQ(0x2fffu, sge.addr); + EXPECT_EQ(1u, sge.len); +} + +TEST(UrmaRuntimeHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge{}; + for (auto values : {std::pair(0xfff, 1), + {0x3000, 1}, {0x2fff, 2}, {0x1000, 0}, + {0x1000, static_cast(UINT32_MAX) + 1}}) { + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, &segment)); + EXPECT_EQ(ERDMAMEM, errno); + } + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); + segment.urma_ctx = reinterpret_cast(1); + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, &segment)); + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = UINT64_MAX - 7; + segment.seg.len = 16; + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); +} + +static urma_target_seg_t g_copy_test_segment; +static int g_copy_test_releases; + +static urma_target_seg_t* ResolveCopyTestSegment(void* address) { + g_copy_test_segment = urma_target_seg_t{}; + g_copy_test_segment.urma_ctx = urma::GetUrmaContext(); + g_copy_test_segment.seg.ubva.va = reinterpret_cast(address); + g_copy_test_segment.seg.len = urma::GetUrmaRecvBlockSize(); + return &g_copy_test_segment; +} + +static urma_target_seg_t* RejectCopyTestSegment(void*) { + return nullptr; +} + +static void DeleteCopyTestData(void* data) { + ++g_copy_test_releases; + delete[] static_cast(data); +} + +TEST(UrmaRuntimeHelperTest, copied_send_coalesces_host_fragments_and_retains_aligned_payload) { + g_copy_test_releases = 0; + butil::IOBuf buffer; + for (const char* text : {"first", "second"}) { + const size_t length = strlen(text); + char* data = new char[length]; + memcpy(data, text, length); + ASSERT_EQ(0, buffer.append_user_data(data, length, DeleteCopyTestData)); + } + ASSERT_EQ(2u, buffer.backing_block_num()); + urma_sge_t sge{}; + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(2, g_copy_test_releases); + EXPECT_EQ("firstsecond", buffer.to_string()); + ASSERT_EQ(1u, buffer.backing_block_num()); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), sge.addr); + EXPECT_EQ(buffer.size(), sge.len); + EXPECT_EQ(&g_copy_test_segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + EXPECT_EQ("firstsecond", std::string(reinterpret_cast(sge.addr), sge.len)); + + buffer.clear(); + const std::string large(urma::GetUrmaRecvBlockSize() - + (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), 'x'); + buffer.append(large); + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(large, buffer.to_string()); + EXPECT_EQ(large.size(), sge.len); +} + +TEST(UrmaRuntimeHelperTest, copied_send_failure_preserves_source_payload) { + butil::IOBuf buffer; + buffer.append("unchanged"); + urma_sge_t sge{}; + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); + EXPECT_EQ(ERDMAMEM, errno); + EXPECT_EQ("unchanged", buffer.to_string()); + const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); + buffer.clear(); + buffer.append(large); + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(EMSGSIZE, errno); + EXPECT_EQ(large, buffer.to_string()); +} + +class UrmaRuntimeMockTest : public ::testing::Test { +protected: + void SetUp() override { + urma::g_skip_urma_init = false; + + urma_init_attr_t init_attr{}; + const urma_status_t status = urma_init(&init_attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + _owns_urma_init = status == URMA_SUCCESS; + + int num_devices = 0; + urma_device_t** devices = urma_get_device_list(&num_devices); + ASSERT_NE(nullptr, devices); + ASSERT_GT(num_devices, 0); + const bool using_mock = + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (!using_mock) { + if (_owns_urma_init) { + EXPECT_EQ(URMA_SUCCESS, urma_uninit()); + _owns_urma_init = false; + } + GTEST_SKIP() << "UrmaRuntimeMockTest requires the URMA link-time mock"; + } + } + + void TearDown() override { + if (_owns_urma_init) { + EXPECT_EQ(URMA_SUCCESS, urma_uninit()); + } + urma::g_skip_urma_init = true; + urma::g_urma_available.store(true, butil::memory_order_relaxed); + } + +private: + bool _owns_urma_init{false}; +}; + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static int ImportPeer(UrmaEndpoint* ep, const ParsedHello& peer) { + return ep->ImportPeer(peer); + } + static void Establish(UrmaEndpoint* ep) { + GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); + ep->_state = UrmaEndpoint::ESTABLISHED; + } + static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { + urma_cr_t cr{}; + cr.status = status; + return ep->HandleCompletion(cr); + } + static void Report(UrmaEndpoint* ep, UrmaFailureReason reason) { + ep->ReportFailure(reason); + } + static void PollFailure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jfc_t invalid_jfc{}; + resource.jfc = &invalid_jfc; + ep->_resource = &resource; + SocketOptions options; + options.user = ep; + SocketId sid; + if (Socket::Create(options, &sid) == 0) { + SocketUniquePtr carrier; + if (Socket::Address(sid, &carrier) == 0) { + const bool polling = FLAGS_urma_use_polling; + FLAGS_urma_use_polling = true; + UrmaEndpoint::PollCq(carrier.get()); + FLAGS_urma_use_polling = polling; + carrier->SetFailed(); + } + } + ep->_resource = nullptr; + resource.jfc = nullptr; + } + static bool Backpressure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_window_size = 0; + ep->_remote_rq_window_size = 1; + butil::IOBuf data; + data.append("payload"); + butil::IOBuf* buf = &data; + const ssize_t rc = ep->CutFromIOBufList(&buf, 1); + const bool ok = rc == -1 && errno == EAGAIN && data.size() == 7; + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return ok; + } + static int FailedAck(UrmaEndpoint* ep) { + // A Jetty without an imported target fails provider submission. + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_imm_window_size = 1; + ep->_remote_window_capacity = 2; + ep->_new_rq_wrs = 2; + urma_cr_t cr{}; + cr.status = URMA_CR_SUCCESS; + cr.user_ctx = 0; + const int rc = ep->HandleCompletion(cr); + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return rc; + } +}; + +} // namespace urma +} // namespace brpc + +class UrmaRecoveryTest : public UrmaRuntimeMockTest { +protected: + void SetUp() override { + UrmaRuntimeMockTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + urma::g_skip_urma_init = true; + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + urma::GlobalUrmaRecoveryPolicy().ReportPortState(true); + } + void TearDown() override { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + UrmaRuntimeMockTest::TearDown(); + } + void CreateSocket(SocketUniquePtr* socket) { + SocketOptions opt; + butil::str2endpoint("127.0.0.1:8123", &opt.remote_side); + opt.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(opt, &sid)); + ASSERT_EQ(0, Socket::Address(sid, socket)); + } +}; + +TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { + ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_TRUE(rdma::GlobalRdmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_EQ(rdma::RdmaRecoveryMode::LEGACY, rdma::GetRdmaRecoveryMode()); + EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); + EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); + EXPECT_NE(std::string::npos, + bvar::Variable::describe_exposed("urma_recovery_status").find("tcp_only")); + EXPECT_FALSE(bvar::Variable::describe_exposed( + "urma_recovery_tcp_fallback_admissions").empty()); +} + +TEST_F(UrmaRecoveryTest, rejects_mismatched_path_before_importing_resources) { + urma::UrmaEndpoint endpoint(nullptr); + urma::ParsedHello peer; + peer.tp_type = urma::GetUrmaTpType() == URMA_CTP ? URMA_RTP : URMA_CTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); + peer.tp_type = URMA_UTP; + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); + EXPECT_EQ(EPROTO, errno); +} + +TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { + for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, + URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); + const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR ? + urma::UrmaFailureReason::RETRY_EXHAUSTED : + status == URMA_CR_RNR_RETRY_CNT_EXC_ERR ? urma::UrmaFailureReason::RNR : + status == URMA_CR_LOC_ACCESS_ERR ? urma::UrmaFailureReason::ACCESS_ERROR : + urma::UrmaFailureReason::TRANSPORT_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + urma::GetUrmaRecoveryStatus().failures[static_cast(reason)]); + EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)], + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); + socket->SetFailed(); + } +} + +TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + urma::UrmaRecoveryTestPeer::PollFailure(&ep); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + 1, + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); +} + +class UrmaRecoveryEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, + ::test::EchoResponse* response, google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + UrmaRecoveryEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + Channel channel; + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 2000; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + ::test::EchoService_Stub stub(&channel); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("recovery over TCP"); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketOptions failed_options; + failed_options.remote_side = server.listen_address(); + failed_options.user = get_client_side_messenger(); + SocketId failed_sid; + ASSERT_EQ(0, Socket::Create(failed_options, &failed_sid)); + SocketUniquePtr failed_socket; + ASSERT_EQ(0, Socket::Address(failed_sid, &failed_socket)); + { + urma::UrmaEndpoint failed_ep(failed_socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&failed_ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( + &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); + } + failed_socket->SetFailed(); + const auto before = urma::GetUrmaRecoveryStatus(); + cntl.Reset(); + response.Clear(); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(before.peer_cooldown_admissions + 1, + urma::GetUrmaRecoveryStatus().peer_cooldown_admissions); + server.Stop(0); + server.Join(); +} + +TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + Server server; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + for (int version : {2, 3}) { + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr)); + ASSERT_GE(fd, 0); + timeval timeout{2, 0}; + ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); + urma::v2_wire::HelloMessage hello; + hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; + hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; + hello.impl_ver = urma::v2_wire::IMPL_V2_VERSION; + hello.buffer_size = 8192; + hello.recv_buffer_cnt = 127; + hello.jetty_id = 42; + memset(hello.eid, 1, 16); + memset(hello.seg_eid, 1, 16); + hello.tp_type = URMA_CTP; + hello.seg_va = 0x100000; + hello.seg_len = 1024 * 1024; + if (version == 2) { + uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; + memcpy(packet, "URMA", 4); + hello.Serialize(packet + 4); + ASSERT_EQ(sizeof(packet), static_cast(write(fd, packet, sizeof(packet)))); + uint8_t reply[sizeof(packet)]; + ASSERT_EQ(sizeof(reply), static_cast(recv(fd, reply, sizeof(reply), MSG_WAITALL))); + EXPECT_EQ(0, memcmp(reply, "URMA", 4)); + urma::v2_wire::HelloMessage rejected; + rejected.Deserialize(reply + 4); + EXPECT_EQ(0, rejected.hello_ver); + EXPECT_EQ(0u, rejected.jetty_id); + } else { + urma::UrmaHello msg; + msg.set_buffer_size(8192); + msg.set_recv_buffer_cnt(127); + msg.set_jetty_id(42); + msg.set_uasid(0); + msg.set_seg_uasid(0); + msg.set_seg_token_id(0); + msg.set_eid(hello.eid, 16); + msg.set_seg_eid(hello.seg_eid, 16); + msg.set_tp_type(URMA_CTP); + msg.set_seg_va(hello.seg_va); + msg.set_seg_len(hello.seg_len); + const std::string body = msg.SerializeAsString(); + uint32_t size = butil::HostToNet32(body.size()); + std::string packet("URM3", 4); + packet.append(reinterpret_cast(&size), 4); + packet.append(body); + ASSERT_EQ(packet.size(), static_cast(write(fd, packet.data(), packet.size()))); + char header[8]; + ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); + EXPECT_EQ(0, memcmp(header, "URM3", 4)); + memcpy(&size, header + 4, 4); + size = butil::NetToHost32(size); + ASSERT_LE(size, 4096u); + std::string reply(size, '\0'); + ASSERT_EQ(size, static_cast(recv(fd, &reply[0], size, MSG_WAITALL))); + urma::UrmaHello rejected; + ASSERT_TRUE(rejected.ParseFromString(reply)); + EXPECT_EQ(0u, rejected.jetty_id()); + } + const std::string request = "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), request.size()))); + char response[4096]; + const ssize_t nr = recv(fd, response, sizeof(response), 0); + ASSERT_GT(nr, 0); + EXPECT_NE(std::string::npos, std::string(response, nr).find(" 200 ")); + } + server.Stop(0); + server.Join(); +} + +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); +#if BRPC_WITH_URMA + urma::g_skip_urma_init = true; + urma::g_urma_available.store(true, butil::memory_order_relaxed); +#endif + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index 7372e41c45..d27367a504 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -16,8 +16,6 @@ // under the License. #include -#include -#include #include #include #include @@ -30,15 +28,6 @@ #include "brpc/urma/urma_handshake.pb.h" #include "brpc/urma/urma_helper.h" #include "urma_types.h" -#include "brpc/channel.h" -#include "brpc/controller.h" -#include "brpc/server.h" -#include "brpc/input_messenger.h" -#include "brpc/urma/urma_endpoint.h" -#include "brpc/urma/urma_recovery_policy.h" -#include "butil/fd_guard.h" -#include "bvar/variable.h" -#include "echo.pb.h" using namespace brpc; @@ -272,181 +261,6 @@ TEST(UrmaHelperTest, selects_priority_matching_transport_path_type) { EXPECT_EQ(3, urma::FindUrmaPriorityForTpType(attr, URMA_RTP)); EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); - attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; - EXPECT_EQ(6, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); - EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( - attr, static_cast(255))); -} - -TEST(UrmaHelperTest, auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { - urma_device_attr_t attr{}; - attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; - attr.dev_cap.priority_info[0].SL = 2; - urma_tp_type_t path = URMA_CTP; - uint8_t priority = 6; - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); - EXPECT_EQ(URMA_RTP, path); - EXPECT_EQ(0, priority); - attr.dev_cap.priority_info[7].tp_type.bs.ctp = 1; - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); - EXPECT_EQ(URMA_CTP, path); - EXPECT_EQ(7, priority); -} - -TEST(UrmaHelperTest, explicit_path_uses_matching_priority_and_combined_capabilities) { - urma_device_attr_t attr{}; - attr.dev_cap.priority_info[2].tp_type.bs.rtp = 1; - attr.dev_cap.priority_info[2].tp_type.bs.ctp = 1; - attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; - urma_tp_type_t path = URMA_UTP; - uint8_t priority = 15; - for (const auto& requested : {"rtp", "ctp"}) { - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); - EXPECT_EQ(std::string(requested) == "rtp" ? URMA_RTP : URMA_CTP, path); - EXPECT_EQ(2, priority); - } -} - -TEST(UrmaHelperTest, rejects_unsupported_paths_in_a_reported_capability_table) { - urma_device_attr_t attr{}; - attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; - urma_tp_type_t path = URMA_UTP; - uint8_t priority = 15; - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "ctp", &path, &priority)); - EXPECT_EQ(URMA_UTP, path); - EXPECT_EQ(15, priority); - attr.dev_cap.priority_info[0].tp_type.value = 0; - attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); -} - -TEST(UrmaHelperTest, providers_without_path_capabilities_keep_compatibility_priorities) { - urma_device_attr_t attr{}; - urma_tp_type_t path = URMA_UTP; - uint8_t priority = 15; - for (const auto& requested : {"auto", "ctp", "rtp"}) { - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); - const bool rtp = std::string(requested) == "rtp"; - EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); - EXPECT_EQ(rtp ? 0 : 6, priority); - } -} - -TEST(UrmaHelperTest, rejects_invalid_path_configuration) { - urma_device_attr_t attr{}; - urma_tp_type_t path = URMA_CTP; - uint8_t priority = 6; - for (const auto& requested : {"", "utp", "RM", "typo"}) { - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); - } - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, nullptr)); -} - -TEST(UrmaHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { - urma_target_seg_t segment{}; - segment.urma_ctx = urma::GetUrmaContext(); - segment.seg.ubva.va = 0x1000; - segment.seg.len = 8192; - urma_sge_t sge; - memset(&sge, 0xff, sizeof(sge)); - ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x1000, 8192, &segment)); - EXPECT_EQ(0x1000u, sge.addr); - EXPECT_EQ(8192u, sge.len); - EXPECT_EQ(&segment, sge.tseg); - EXPECT_EQ(nullptr, sge.user_tseg); - ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x2fff, 1, &segment)); - EXPECT_EQ(0x2fffu, sge.addr); - EXPECT_EQ(1u, sge.len); -} - -TEST(UrmaHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { - urma_target_seg_t segment{}; - segment.urma_ctx = urma::GetUrmaContext(); - segment.seg.ubva.va = 0x1000; - segment.seg.len = 8192; - urma_sge_t sge{}; - for (auto values : {std::pair(0xfff, 1), - {0x3000, 1}, {0x2fff, 2}, {0x1000, 0}, - {0x1000, static_cast(UINT32_MAX) + 1}}) { - EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, &segment)); - EXPECT_EQ(ERDMAMEM, errno); - } - EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); - segment.urma_ctx = reinterpret_cast(1); - EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, &segment)); - segment.urma_ctx = urma::GetUrmaContext(); - segment.seg.ubva.va = UINT64_MAX - 7; - segment.seg.len = 16; - EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); -} - -static urma_target_seg_t g_copy_test_segment; -static int g_copy_test_releases; - -static urma_target_seg_t* ResolveCopyTestSegment(void* address) { - g_copy_test_segment = urma_target_seg_t{}; - g_copy_test_segment.urma_ctx = urma::GetUrmaContext(); - g_copy_test_segment.seg.ubva.va = reinterpret_cast(address); - g_copy_test_segment.seg.len = urma::GetUrmaRecvBlockSize(); - return &g_copy_test_segment; -} - -static urma_target_seg_t* RejectCopyTestSegment(void*) { - return nullptr; -} - -static void DeleteCopyTestData(void* data) { - ++g_copy_test_releases; - delete[] static_cast(data); -} - -TEST(UrmaHelperTest, copied_send_coalesces_host_fragments_and_retains_aligned_payload) { - g_copy_test_releases = 0; - butil::IOBuf buffer; - for (const char* text : {"first", "second"}) { - const size_t length = strlen(text); - char* data = new char[length]; - memcpy(data, text, length); - ASSERT_EQ(0, buffer.append_user_data(data, length, DeleteCopyTestData)); - } - ASSERT_EQ(2u, buffer.backing_block_num()); - urma_sge_t sge{}; - ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); - EXPECT_EQ(2, g_copy_test_releases); - EXPECT_EQ("firstsecond", buffer.to_string()); - ASSERT_EQ(1u, buffer.backing_block_num()); - EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); - EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), sge.addr); - EXPECT_EQ(buffer.size(), sge.len); - EXPECT_EQ(&g_copy_test_segment, sge.tseg); - EXPECT_EQ(nullptr, sge.user_tseg); - EXPECT_EQ("firstsecond", std::string(reinterpret_cast(sge.addr), sge.len)); - - buffer.clear(); - const std::string large(urma::GetUrmaRecvBlockSize() - - (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), 'x'); - buffer.append(large); - ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); - EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); - EXPECT_EQ(large, buffer.to_string()); - EXPECT_EQ(large.size(), sge.len); -} - -TEST(UrmaHelperTest, copied_send_failure_preserves_source_payload) { - butil::IOBuf buffer; - buffer.append("unchanged"); - urma_sge_t sge{}; - EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); - EXPECT_EQ(ERDMAMEM, errno); - EXPECT_EQ("unchanged", buffer.to_string()); - const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); - buffer.clear(); - buffer.append(large); - EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); - EXPECT_EQ(EMSGSIZE, errno); - EXPECT_EQ(large, buffer.to_string()); } // --------------------------------------------------------------------------- @@ -779,345 +593,6 @@ TEST_F(UrmaMockTest, urma_free_device_list(devices); } - -namespace brpc { -namespace urma { - -class UrmaRecoveryTestPeer { -public: - static int ImportPeer(UrmaEndpoint* ep, const ParsedHello& peer) { - return ep->ImportPeer(peer); - } - static void Establish(UrmaEndpoint* ep) { - GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); - ep->_state = UrmaEndpoint::ESTABLISHED; - } - static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { - urma_cr_t cr{}; - cr.status = status; - return ep->HandleCompletion(cr); - } - static void Report(UrmaEndpoint* ep, UrmaFailureReason reason) { - ep->ReportFailure(reason); - } - static void PollFailure(UrmaEndpoint* ep) { - UrmaResource resource; - urma_jfc_t invalid_jfc{}; - resource.jfc = &invalid_jfc; - ep->_resource = &resource; - SocketOptions options; - options.user = ep; - SocketId sid; - if (Socket::Create(options, &sid) == 0) { - SocketUniquePtr carrier; - if (Socket::Address(sid, &carrier) == 0) { - const bool polling = FLAGS_urma_use_polling; - FLAGS_urma_use_polling = true; - UrmaEndpoint::PollCq(carrier.get()); - FLAGS_urma_use_polling = polling; - carrier->SetFailed(); - } - } - ep->_resource = nullptr; - resource.jfc = nullptr; - } - static bool Backpressure(UrmaEndpoint* ep) { - UrmaResource resource; - urma_jetty_t jetty{}; - urma_target_jetty_t remote{}; - resource.jetty = &jetty; - resource.remote_jetty = &remote; - ep->_resource = &resource; - ep->_sq_window_size = 0; - ep->_remote_rq_window_size = 1; - butil::IOBuf data; - data.append("payload"); - butil::IOBuf* buf = &data; - const ssize_t rc = ep->CutFromIOBufList(&buf, 1); - const bool ok = rc == -1 && errno == EAGAIN && data.size() == 7; - ep->_resource = nullptr; - resource.jetty = nullptr; - resource.remote_jetty = nullptr; - return ok; - } - static int FailedAck(UrmaEndpoint* ep) { - // A Jetty without an imported target fails provider submission. - UrmaResource resource; - urma_jetty_t jetty{}; - urma_target_jetty_t remote{}; - resource.jetty = &jetty; - resource.remote_jetty = &remote; - ep->_resource = &resource; - ep->_sq_imm_window_size = 1; - ep->_remote_window_capacity = 2; - ep->_new_rq_wrs = 2; - urma_cr_t cr{}; - cr.status = URMA_CR_SUCCESS; - cr.user_ctx = 0; - const int rc = ep->HandleCompletion(cr); - ep->_resource = nullptr; - resource.jetty = nullptr; - resource.remote_jetty = nullptr; - return rc; - } -}; - -} // namespace urma -} // namespace brpc - -class UrmaRecoveryTest : public UrmaMockTest { -protected: - void SetUp() override { - UrmaMockTest::SetUp(); - if (IsSkipped() || HasFatalFailure()) { - return; - } - urma::g_skip_urma_init = true; - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - urma::GlobalUrmaRecoveryPolicy().ReportPortState(true); - } - void TearDown() override { - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - UrmaMockTest::TearDown(); - } - void CreateSocket(SocketUniquePtr* socket) { - SocketOptions opt; - butil::str2endpoint("127.0.0.1:8123", &opt.remote_side); - opt.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(opt, &sid)); - ASSERT_EQ(0, Socket::Address(sid, socket)); - } -}; - -TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { - ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); - EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); - EXPECT_TRUE(rdma::GlobalRdmaRecoveryPolicy().AllowNewConnection("peer")); - EXPECT_EQ(rdma::RdmaRecoveryMode::LEGACY, rdma::GetRdmaRecoveryMode()); - EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); - EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); - EXPECT_NE(std::string::npos, - bvar::Variable::describe_exposed("urma_recovery_status").find("tcp_only")); - EXPECT_FALSE(bvar::Variable::describe_exposed( - "urma_recovery_tcp_fallback_admissions").empty()); -} - -TEST_F(UrmaRecoveryTest, rejects_mismatched_path_before_importing_resources) { - urma::UrmaEndpoint endpoint(nullptr); - urma::ParsedHello peer; - peer.tp_type = urma::GetUrmaTpType() == URMA_CTP ? URMA_RTP : URMA_CTP; - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); - EXPECT_EQ(EPROTO, errno); - peer.tp_type = URMA_UTP; - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::ImportPeer(&endpoint, peer)); - EXPECT_EQ(EPROTO, errno); -} - -TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { - for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, - URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - urma::UrmaEndpoint ep(socket.get()); - urma::UrmaRecoveryTestPeer::Establish(&ep); - const auto before = urma::GetUrmaRecoveryStatus(); - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); - const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR ? - urma::UrmaFailureReason::RETRY_EXHAUSTED : - status == URMA_CR_RNR_RETRY_CNT_EXC_ERR ? urma::UrmaFailureReason::RNR : - status == URMA_CR_LOC_ACCESS_ERR ? urma::UrmaFailureReason::ACCESS_ERROR : - urma::UrmaFailureReason::TRANSPORT_ERROR; - EXPECT_EQ(before.failures[static_cast(reason)] + 1, - urma::GetUrmaRecoveryStatus().failures[static_cast(reason)]); - EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, - urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); - EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)], - urma::GetUrmaRecoveryStatus().failures[ - static_cast(urma::UrmaFailureReason::CQ_POLL)]); - socket->SetFailed(); - } -} - -TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - urma::UrmaEndpoint ep(socket.get()); - urma::UrmaRecoveryTestPeer::Establish(&ep); - EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); - EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); - urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); - EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - socket->SetFailed(); -} - -TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - urma::UrmaEndpoint ep(socket.get()); - urma::UrmaRecoveryTestPeer::Establish(&ep); - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); - EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - socket->SetFailed(); -} - -TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - urma::UrmaEndpoint ep(socket.get()); - urma::UrmaRecoveryTestPeer::Establish(&ep); - const auto before = urma::GetUrmaRecoveryStatus(); - urma::UrmaRecoveryTestPeer::PollFailure(&ep); - EXPECT_TRUE(socket->Failed()); - EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + 1, - urma::GetUrmaRecoveryStatus().failures[ - static_cast(urma::UrmaFailureReason::CQ_POLL)]); -} - -class UrmaRecoveryEcho : public ::test::EchoService { - void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, - ::test::EchoResponse* response, google::protobuf::Closure* done) override { - response->set_message(request->message()); - done->Run(); - } -}; - -TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); - UrmaRecoveryEcho service; - Server server; - ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); - ServerOptions so; - so.socket_mode = SOCKET_MODE_URMA; - ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); - Channel channel; - ChannelOptions co; - co.socket_mode = SOCKET_MODE_URMA; - co.connection_type = "short"; - co.timeout_ms = 2000; - ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); - ::test::EchoService_Stub stub(&channel); - Controller cntl; - ::test::EchoRequest request; - ::test::EchoResponse response; - request.set_message("recovery over TCP"); - stub.Echo(&cntl, &request, &response, nullptr); - EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); - EXPECT_EQ(request.message(), response.message()); - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); - SocketOptions failed_options; - failed_options.remote_side = server.listen_address(); - failed_options.user = get_client_side_messenger(); - SocketId failed_sid; - ASSERT_EQ(0, Socket::Create(failed_options, &failed_sid)); - SocketUniquePtr failed_socket; - ASSERT_EQ(0, Socket::Address(failed_sid, &failed_socket)); - { - urma::UrmaEndpoint failed_ep(failed_socket.get()); - urma::UrmaRecoveryTestPeer::Establish(&failed_ep); - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( - &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); - } - failed_socket->SetFailed(); - const auto before = urma::GetUrmaRecoveryStatus(); - cntl.Reset(); - response.Clear(); - stub.Echo(&cntl, &request, &response, nullptr); - EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); - EXPECT_EQ(request.message(), response.message()); - EXPECT_EQ(before.peer_cooldown_admissions + 1, - urma::GetUrmaRecoveryStatus().peer_cooldown_admissions); - server.Stop(0); - server.Join(); -} - -TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { - urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); - Server server; - ServerOptions so; - so.socket_mode = SOCKET_MODE_URMA; - ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); - for (int version : {2, 3}) { - butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr)); - ASSERT_GE(fd, 0); - timeval timeout{2, 0}; - ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); - urma::v2_wire::HelloMessage hello; - hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; - hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; - hello.impl_ver = urma::v2_wire::IMPL_V2_VERSION; - hello.buffer_size = 8192; - hello.recv_buffer_cnt = 127; - hello.jetty_id = 42; - memset(hello.eid, 1, 16); - memset(hello.seg_eid, 1, 16); - hello.tp_type = URMA_CTP; - hello.seg_va = 0x100000; - hello.seg_len = 1024 * 1024; - if (version == 2) { - uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; - memcpy(packet, "URMA", 4); - hello.Serialize(packet + 4); - ASSERT_EQ(sizeof(packet), static_cast(write(fd, packet, sizeof(packet)))); - uint8_t reply[sizeof(packet)]; - ASSERT_EQ(sizeof(reply), static_cast(recv(fd, reply, sizeof(reply), MSG_WAITALL))); - EXPECT_EQ(0, memcmp(reply, "URMA", 4)); - urma::v2_wire::HelloMessage rejected; - rejected.Deserialize(reply + 4); - EXPECT_EQ(0, rejected.hello_ver); - EXPECT_EQ(0u, rejected.jetty_id); - } else { - urma::UrmaHello msg; - msg.set_buffer_size(8192); - msg.set_recv_buffer_cnt(127); - msg.set_jetty_id(42); - msg.set_uasid(0); - msg.set_seg_uasid(0); - msg.set_seg_token_id(0); - msg.set_eid(hello.eid, 16); - msg.set_seg_eid(hello.seg_eid, 16); - msg.set_tp_type(URMA_CTP); - msg.set_seg_va(hello.seg_va); - msg.set_seg_len(hello.seg_len); - const std::string body = msg.SerializeAsString(); - uint32_t size = butil::HostToNet32(body.size()); - std::string packet("URM3", 4); - packet.append(reinterpret_cast(&size), 4); - packet.append(body); - ASSERT_EQ(packet.size(), static_cast(write(fd, packet.data(), packet.size()))); - char header[8]; - ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); - EXPECT_EQ(0, memcmp(header, "URM3", 4)); - memcpy(&size, header + 4, 4); - size = butil::NetToHost32(size); - ASSERT_LE(size, 4096u); - std::string reply(size, '\0'); - ASSERT_EQ(size, static_cast(recv(fd, &reply[0], size, MSG_WAITALL))); - urma::UrmaHello rejected; - ASSERT_TRUE(rejected.ParseFromString(reply)); - EXPECT_EQ(0u, rejected.jetty_id()); - } - const std::string request = "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; - ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), request.size()))); - char response[4096]; - const ssize_t nr = recv(fd, response, sizeof(response), 0); - ASSERT_GT(nr, 0); - EXPECT_NE(std::string::npos, std::string(response, nr).find(" 200 ")); - } - server.Stop(0); - server.Join(); -} - #else // BRPC_WITH_URMA // When URMA is not compiled in, the test file is a no-op so the build stays From e7ba01f266e4a627fe23eee4090ac76446ace3cb Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:10:25 -0700 Subject: [PATCH 21/29] Support RTP business SL mapping and isolate QoS unit tests --- docs/cn/rdma.md | 27 +-- docs/cn/urma.md | 68 +++----- docs/en/rdma.md | 32 ++-- docs/en/urma.md | 92 +++++----- src/brpc/urma/mock_urma.cpp | 2 + src/brpc/urma/urma_endpoint.cpp | 12 +- src/brpc/urma/urma_helper.cpp | 96 ++++++++--- src/brpc/urma/urma_helper.h | 19 ++- src/brpc/urma/urma_qos.h | 2 +- test/CMakeLists.txt | 6 + test/brpc_rdma_qos_unittest.cpp | 286 ++++++++++++++++++++++++++++++++ test/brpc_rdma_unittest.cpp | 143 ---------------- test/brpc_urma_qos_unittest.cpp | 205 +++++++++++++++++++++-- 13 files changed, 658 insertions(+), 332 deletions(-) create mode 100644 test/brpc_rdma_qos_unittest.cpp diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index f000b52fb9..631f1a6e83 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,6 +1,6 @@ # 业务级 QoS -通过 `ChannelOptions.rdma_qos` 配置请求方向的 RDMA QoS: +请求方向配置 `ChannelOptions.rdma_qos`,响应方向配置 `ServerOptions.rdma_qos`: ```cpp brpc::ChannelOptions options; @@ -9,28 +9,15 @@ options.rdma_qos.service_level = 3; options.rdma_qos.traffic_class = (26 << 2) | 2; // DSCP=26, ECN=2 ``` -`service_level` 范围为 0–15,`traffic_class` 为完整的 8 位 GRH/IP Traffic Class,范围为 0–255, -两者默认均为 0。SL 的调度待遇由网络侧 SL→VL 或优先级映射决定。 -RoCE 部署结合网卡及交换机的 PCP/DSCP 映射配置;示例中的 DSCP/ECN 值按部署策略选择。 +`service_level` 范围为 0–15,`traffic_class` 是完整的 8 位 GRH/IP 字节,范围为 0–255, +两者默认均为 0。网络配置决定 SL→VL 及 PCP/DSCP 映射和调度策略。 -同一地址下,相同 QoS 的 Channel 可共享连接,不同 SL 或 Traffic Class 使用独立连接及 QP。 -配置随单连接、连接池和短连接传递,并在 QP 重建时重新应用。 - -响应方向通过 `ServerOptions.rdma_qos` 配置: - -```cpp -brpc::ServerOptions options; -options.socket_mode = brpc::SOCKET_MODE_RDMA; -options.rdma_qos.service_level = 3; -options.rdma_qos.traffic_class = 106; -``` - -客户端和服务端分别配置各自的发送方向。设置在 Channel 初始化或 Server 启动时生效, -V2/V3 握手沿用现有格式;TCP 传输按原配置运行。 +不同 QoS 使用独立连接与 QP,相同配置共享连接。配置随服务发现、连接池、短连接和 +服务端 accept 传递,在 QP 重建时重新应用。v2/v3 握手沿用现有格式。 连接调试信息提供 `rdma_sl` 和 `rdma_traffic_class`。 -`example/rdma_performance` 的 client/server 支持 `--rdma_sl`、`--rdma_traffic_class`。 -实机验证时检查 QP 属性和实际报文标记,再比较竞争流量下的吞吐与 P99。 +`example/rdma_performance` 的两端支持 `--rdma_sl`、`--rdma_traffic_class`。 +实机检查 QP 属性与报文标记,再结合竞争流量比较吞吐和 P99。 # 编译 diff --git a/docs/cn/urma.md b/docs/cn/urma.md index c08cd5b636..bd53961dd2 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -159,6 +159,7 @@ block 都由注册 segment 支撑,发送路径可直接从 IOBuf block refs | `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 | | `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | `--urma_device` | "" | URMA 设备名(空=首个) | +| `--urma_tp_type` | auto | RM 路径:auto、ctp、rtp | | `--urma_max_sge` | 0 | 每 WR SGE 上限(0=设备上限) | | `--urma_bonding_mode` | 0 | bonding 模式:0=standalone,1=active-backup,2=balance | | `--urma_bonding_level` | 0 | bonding 层级:0=IODIE,1=port | @@ -196,60 +197,43 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径,而 UBRing ## 业务 SL 与优先级 -`ChannelOptions.urma_qos` 控制请求发送方向, -`ServerOptions.urma_qos` 控制响应发送方向: +`ChannelOptions.urma_qos` 控制请求发送方向,`ServerOptions.urma_qos` 控制响应发送方向: ```cpp brpc::ChannelOptions client_options; client_options.socket_mode = brpc::SOCKET_MODE_URMA; -client_options.urma_qos.service_level = 6; +client_options.urma_qos.service_level = 2; brpc::ServerOptions server_options; server_options.socket_mode = brpc::SOCKET_MODE_URMA; -server_options.urma_qos.service_level = 7; +server_options.urma_qos.service_level = 2; ``` -当前使用 RM+CTP。CTP 的 SL 由设备 `priority_info[priority].SL` 决定, -业务 SL 与 priority 数字可以不同。`service_level` 默认为 -1,表示不指定 SL; -显式范围为 0–255。指定 SL 后,选择首个同时匹配该 SL 且包含 CTP 能力的 priority。 -`priority` 可单独指定,范围为 0–15;同时指定 SL 和 priority 时,两者必须匹配设备映射。 -找不到 CTP 映射或配置冲突时,Channel 初始化和 Server 启动报错,列出支持的 priority:SL。 -例如设备只有 priority 6→SL 6、priority 7→SL 7 支持 CTP 时,SL 10 会被拒绝。 +`--urma_tp_type=auto` 按设备能力优先选择 CTP,其次 RTP;`ctp`、`rtp` 显式选择路径。 +两端使用相同路径,进程初始化后保持固定。设备未提供能力表时,auto/ctp 使用 priority 6, +rtp 使用 priority 0。 -两项均为 -1 时沿用原自动选择:首个 CTP priority,未报告时使用兼容值 6。 -设备全局映射与调度待遇由平台配置;brpc 使用初始化时读取的映射,平台修改后重启进程加载。 -本接口按当前 RM+CTP 路径校验。 +| 配置 | 默认 | 显式范围 | 含义 | +| --- | --- | --- | --- | +| `service_level` | -1 | 0–255 | 选择当前路径下首个映射到该 SL 的 priority | +| `priority` | -1 | 0–15 | 指定当前路径支持的 priority | -不同 SL/priority 配置使用独立连接,相同配置共享连接。配置继承到服务发现、连接池、 -短连接和服务端 accept 路径,重建时重新应用。默认预建池只用于匹配其 priority 的连接, -其他 priority 按需创建 Jetty。连接调试信息显示请求 SL/priority、Jetty priority 和映射 SL。 -请求和响应分别配置各自的发送策略,v2/v3 握手格式沿用现有协议。 +两项均为 -1 时使用设备默认 priority;同时指定时须匹配同一设备映射。 +不支持或冲突时,Channel 初始化、Server 启动报错,并列出当前路径支持的 priority:SL。 +例如 RTP priority 0→SL 2 时配置 `service_level=2`,Jetty 使用 priority 0。 +平台配置设备映射与调度策略,brpc 使用进程初始化时的映射快照。 -`example/urma_performance` 的 client/server 支持 `--urma_sl=6` 和 `--urma_priority=6`。 -示例值须在设备上有匹配的 CTP 映射,数字大小不代表调度优先关系。 +不同配置使用独立连接,相同配置共享连接。配置随服务发现、连接池、短连接和 +服务端 accept 传递,Jetty 重建时重新应用。预建池服务于默认 priority, +其他 priority 按需创建。v2/v3 握手使用所选路径。 -### 实机验证 +### 功能验证 -先使用同一对设备验证原生 RM、CTP、共享 JFR 数据通路。设置本机 `URMA_DEVICE`, -客户端另设 `SERVER_IP`,先启动服务端,再启动客户端: +用 `urma_admin show --dev "$URMA_DEVICE" --whole` 核对当前路径的 priority:SL。 +`example/urma_performance` 的两端均支持 `--urma_tp_type=rtp --urma_sl=2`, +也可通过 `--urma_priority` 指定匹配项。先以默认配置完成 RPC,再分别改变请求和响应 SL, +验证合法映射、不支持的 SL 和 SL/priority 冲突。对 v2/v3、single/pooled 及 polling 模式重复验证。 -```bash -# 服务端 -urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ - -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -# 客户端 -urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ - -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -S "$SERVER_IP" -``` - -两端再将 `-I 0` 改为 `-I 64` 做 inline 对照。原生测试通过后,用 -默认 SL/priority 完成 brpc 单流基线,再逐个测试设备支持的显式 SL, -请求、响应方向分别改变配置,并验证不支持的 SL 和冲突配置在初始化时被拒绝。 -连接调试信息应为 `state=ESTABLISHED`,并显示预期的 `urma_effective_priority` 与 -`urma_effective_sl`;TCP 回落的吞吐数据不计入 URMA QoS 结果。 -最后启动竞争流量,结合设备 SL 映射、队列计数器、吞吐和 P99 验证调度效果。 - -记录两端内核、UMDK、provider 和固件版本。`status=4` 是本地内存访问错误; -原生测试出现该错误时先定位数据通路。brpc 完成错误日志包含收发方向、请求 SL/priority、 -Jetty priority 和映射 SL,初始化日志包含 `default_priority`、`default_sl`。mock 测试验证配置传递和 -Jetty 属性,报文标记与调度效果由实机测试验证。 +连接调试信息提供 `state`、`urma_effective_priority` 和 `urma_effective_sl`。 +核对 URMA 连接为 `ESTABLISHED`、映射符合设备配置、RPC 无错误。 +调度效果测试结合竞争流量、设备队列计数器、吞吐与 P99。 diff --git a/docs/en/rdma.md b/docs/en/rdma.md index cfac9f83f3..19bbe5d0e6 100644 --- a/docs/en/rdma.md +++ b/docs/en/rdma.md @@ -1,20 +1,18 @@ -# Business-level QoS - -Set `ChannelOptions.rdma_qos.service_level` (0–15) and -`ChannelOptions.rdma_qos.traffic_class` (0–255) for outbound requests. -Both default to 0. Traffic Class is the full GRH/IP byte, including DSCP and ECN. -Set `ServerOptions.rdma_qos` independently for outbound responses. - -Different SL or Traffic Class values use separate connections and QPs, while equal -settings can share connections. Settings follow pooled and short sockets and are -applied whenever the QP is brought up. V2/V3 handshake formats stay the same. -TCP transports retain their existing settings. - -The fabric defines how SL maps to VLs and scheduling priority. For RoCE, configure -the NIC and switch PCP/DSCP mappings to match the deployment. The performance -client/server accept `--rdma_sl` and `--rdma_traffic_class`; socket diagnostics show -both applied values. Validate QP attributes and packet markings on hardware before -comparing throughput and P99 under competing workloads. +# Business QoS + +Configure `ChannelOptions.rdma_qos` for outgoing requests and +`ServerOptions.rdma_qos` for outgoing responses. `service_level` accepts 0–15; +`traffic_class` accepts the full 8-bit GRH/IP byte, 0–255, including DSCP and ECN. +Both default to 0. The network configures SL→VL and PCP/DSCP mappings and scheduling. + +Different QoS settings use separate connections and QPs; equal settings share them. +Naming services, pooled and short connections, accepted sockets and rebuilt QPs +inherit the settings. v2/v3 handshake formats stay the same. Connection diagnostics +show `rdma_sl` and `rdma_traffic_class`. + +Both examples in `example/rdma_performance` accept `--rdma_sl` and +`--rdma_traffic_class`. Check QP attributes and packet markings, then compare +throughput and P99 under competing traffic. # Build diff --git a/docs/en/urma.md b/docs/en/urma.md index 5ed721b39b..56e569e222 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -170,6 +170,7 @@ All flags use the `urma_` prefix (mirroring RDMA's `rdma_` prefix): | `--urma_recv_zerocopy` | true | Zero-copy receives above `--urma_zerocopy_min_size` | | `--urma_zerocopy_min_size` | 512 | Receives smaller than this are copied | | `--urma_device` | "" | URMA device name (empty = first) | +| `--urma_tp_type` | auto | RM path: auto, ctp or rtp | | `--urma_max_sge` | 0 | Max SGEs per WR (0 = device max) | | `--urma_bonding_mode` | 0 | Bonding mode: 0=standalone, 1=active-backup, 2=balance | | `--urma_bonding_level` | 0 | Bonding level: 0=IODIE, 1=port | @@ -216,62 +217,45 @@ Set `ChannelOptions.urma_qos` for outgoing requests and ```cpp brpc::ChannelOptions client_options; client_options.socket_mode = brpc::SOCKET_MODE_URMA; -client_options.urma_qos.service_level = 6; +client_options.urma_qos.service_level = 2; brpc::ServerOptions server_options; server_options.socket_mode = brpc::SOCKET_MODE_URMA; -server_options.urma_qos.service_level = 7; +server_options.urma_qos.service_level = 2; ``` -The current path uses RM and CTP. CTP derives SL from -`priority_info[priority].SL`; SL and priority need not have equal numeric values. -`service_level` defaults to -1 (unspecified), with explicit values in [0,255]. -An explicit SL selects the first priority with both that SL and CTP capability. -`priority` can also be set explicitly in [0,15]. When both are specified, the -device mapping must satisfy both. Channel initialization and Server startup -reject unsupported or conflicting requests and report supported priority:SL pairs. -For example, if only priority 6/SL 6 and priority 7/SL 7 support CTP, SL 10 is rejected. - -With both fields at -1, automatic selection retains the first CTP priority and -the existing compatibility value 6 when none is reported. The platform configures -device mappings and scheduling; brpc uses the initialization snapshot. Restart -processes after changing platform mappings. Validation applies to the RM/CTP path. - -Different SL/priority configurations use separate connections; equal configurations -share connections. Naming services, pooled and short connections, accepted sockets -and rebuilt Jettys inherit the configuration. The prepared pool serves matching -priorities; other priorities allocate Jettys on demand. Debug output shows the -requested SL/priority, Jetty priority and mapped SL. Each peer configures its outgoing direction; -v2/v3 handshake formats remain the same. - -The client and server in `example/urma_performance` accept `--urma_sl=6` and -`--urma_priority=6`. Example values require matching CTP mappings on the selected -device; their numeric order does not specify scheduling precedence. - -### Hardware validation - -First test the same device pair with native RM, CTP and shared JFR. Set the local -`URMA_DEVICE` on each host and `SERVER_IP` on the client. Start the server first: - -```bash -# Server -urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ - -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -# Client -urma_perftest send_lat --mgmt_proto tcp -d "$URMA_DEVICE" --ctp \ - -p 0 -j true -I 0 -s 64 -n 1000 -P 8003 -S "$SERVER_IP" -``` - -Repeat with `-I 64` on both hosts for the inline comparison. Once native tests -pass, establish a single-flow brpc baseline with default SL/priority, then test -supported explicit SLs, changing the request and response directions separately. -Verify that unsupported SLs and conflicting settings fail initialization. -Connection debug output must show `state=ESTABLISHED` and the expected -`urma_effective_priority` and `urma_effective_sl`. Exclude TCP fallback measurements from URMA QoS results. -Then compare device SL mappings, queue counters, throughput and P99 under contention. - -Record kernel, UMDK, provider and firmware versions on both hosts. Status 4 means -a local memory access error; resolve native data-path failures before measuring -QoS. brpc completion errors log the direction, requested SL/priority, Jetty priority -and mapped SL; initialization logs `default_priority` and `default_sl`. Mock tests check configuration propagation -and Jetty attributes; hardware tests check packet markings and scheduling effects. +`--urma_tp_type=auto` prefers CTP, then RTP, using device capabilities. +Use `ctp` or `rtp` to select a path explicitly. Both peers use the same path, +fixed at process initialization. Providers without a capability table use +priority 6 for auto/CTP and priority 0 for RTP. + +| Option | Default | Explicit range | Meaning | +| --- | --- | --- | --- | +| `service_level` | -1 | 0–255 | Select the first priority mapping to this SL on the selected path | +| `priority` | -1 | 0–15 | Select a priority supported by the selected path | + +Both fields at -1 select the device default. When both are explicit, they must +match the same device mapping. Channel initialization and Server startup reject +unsupported or conflicting settings and report supported priority:SL pairs. +For example, RTP priority 0 mapping to SL 2 resolves `service_level=2` to priority 0. +The platform configures mappings and scheduling; brpc uses the initialization snapshot. + +Different configurations use separate connections; equal configurations share +connections. Naming services, pooled and short connections, accepted sockets and +rebuilt Jettys inherit the configuration. The prepared pool serves the default +priority; other priorities allocate Jettys on demand. Both v2/v3 handshakes carry +the selected path. + +### Functional validation + +Inspect the selected path's priority:SL map with +`urma_admin show --dev "$URMA_DEVICE" --whole`. +Both examples in `example/urma_performance` accept `--urma_tp_type=rtp --urma_sl=2` +and an optional matching `--urma_priority`. Establish RPCs with defaults, then +change request and response SLs separately. Check supported mappings, unsupported +SLs and conflicting pairs across v2/v3, single/pooled and polling modes. + +Connection diagnostics expose `state`, `urma_effective_priority` and +`urma_effective_sl`. Check that URMA is `ESTABLISHED`, the mapping matches the +device and RPCs succeed. Evaluate scheduling with competing traffic, device queue +counters, throughput and P99. diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index 853b8773f6..742531788a 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -243,8 +243,10 @@ urma_status_t urma_query_device(urma_device_t *device, mock_device_attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; mock_device_attr.dev_cap.priority_info[3].SL = 3; mock_device_attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; mock_device_attr.dev_cap.priority_info[6].SL = 2; mock_device_attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + mock_device_attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; mock_device_attr.dev_cap.priority_info[10].SL = 4; mock_device_attr.dev_cap.max_jfc = 1024; mock_device_attr.dev_cap.max_jetty = 1024; diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 52864970ef..316b8ce04c 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -262,7 +262,7 @@ void UrmaEndpoint::MakeLocalParsedHello(ParsedHello* out) const { : _resource->jetty->jetty_id.eid.raw; std::memcpy(out->eid, advertised_eid, 16); } - out->tp_type = static_cast(URMA_CTP); + out->tp_type = static_cast(GetUrmaTpType()); // Pool segment: flatten g_pool_seg's seg fields. urma_target_seg_t* pool = GetPoolSegFor(nullptr); if (pool) { @@ -544,6 +544,12 @@ void UrmaEndpoint::DeallocateResources() { // ============================================================================ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { + if (peer.tp_type != static_cast(GetUrmaTpType())) { + LOG(ERROR) << "URMA path mismatch: local=" << GetUrmaTpType() + << " remote=" << static_cast(peer.tp_type); + errno = EPROTO; + return -1; + } urma_context_t* ctx = GetUrmaContext(); if (!ctx) { errno = ENODEV; @@ -577,10 +583,6 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { remote.jetty_id.id = peer.jetty_id; remote.trans_mode = URMA_TM_RM; remote.type = URMA_JETTY; - if (peer.tp_type > static_cast(URMA_UTP)) { - errno = EPROTO; - return -1; - } remote.tp_type = static_cast(peer.tp_type); urma_token_t token{}; diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index 3995089f4b..1bcf5c7e6c 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -88,6 +88,8 @@ DEFINE_int32(urma_bonding_mode, 0, "1=active-backup, 2=balance."); DEFINE_int32(urma_bonding_level, 0, "Bonding level for bonding devices: 0=IODIE, 1=port."); +DEFINE_string(urma_tp_type, "auto", "RM path: auto, ctp or rtp"); + DEFINE_int32(urma_prepared_jetty_cnt, 8, "Requested number of pre-allocated Jetty+CQ sets for fast " "connect; capped automatically according to RLIMIT_NOFILE"); @@ -121,8 +123,7 @@ static urma_device_attr_t g_device_attr{}; static int g_max_sge = 1; static size_t g_recv_block_size = 8 * 1024; static bool g_is_bonding_device = false; -// Prefer the device capability table and retain priority 6 as a compatibility -// fallback for CTP providers that do not report a priority. +static urma_tp_type_t g_tp_type = URMA_CTP; static uint8_t g_jetty_priority = 6; // urma_init/urma_uninit manage process-global liburma state. Only uninitialize // it when this helper performed the successful initialization; URMA_EEXIST @@ -526,14 +527,13 @@ static bool GlobalUrmaInitializeImpl() { g_device = nullptr; return false; } - const int ctp_priority = - FindUrmaPriorityForTpType(g_device_attr, URMA_CTP); - if (ctp_priority >= 0) { - g_jetty_priority = static_cast(ctp_priority); - } else { - LOG(WARNING) << "URMA device does not report a CTP priority; " - "falling back to compatibility priority " - << static_cast(g_jetty_priority); + if (!SelectUrmaTransportPath(g_device_attr, FLAGS_urma_tp_type, + &g_tp_type, &g_jetty_priority)) { + LOG(ERROR) << "URMA path selection failed: device=" << device_name + << " requested=" << FLAGS_urma_tp_type; + urma_free_device_list(devices); + g_device = nullptr; + return false; } uint32_t eid_cnt = 0; urma_eid_info_t* eids = urma_get_eid_list(found, &eid_cnt); @@ -629,6 +629,7 @@ static bool GlobalUrmaInitializeImpl() { // resources on exit; GlobalRelease remains available for init rollback. LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device + << " tp_type=" << (g_tp_type == URMA_CTP ? "ctp" : "rtp") << " default_priority=" << static_cast(g_jetty_priority) << " default_sl=" << GetUrmaServiceLevelForPriority(g_jetty_priority) << " max_sge=" << g_max_sge @@ -689,19 +690,52 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, } return -1; } +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority) { + if (!tp_type || !priority || + (requested != "auto" && requested != "ctp" && requested != "rtp")) { + return false; + } + for (auto candidate : {URMA_CTP, URMA_RTP}) { + if ((requested == "ctp" && candidate != URMA_CTP) || + (requested == "rtp" && candidate != URMA_RTP)) { + continue; + } + const int match = FindUrmaPriorityForTpType(attr, candidate); + if (match >= 0) { + *tp_type = candidate; + *priority = static_cast(match); + return true; + } + } + for (int i = 0; i <= URMA_MAX_PRIORITY; ++i) { + if (attr.dev_cap.priority_info[i].tp_type.value != 0) { + return false; + } + } + *tp_type = requested == "rtp" ? URMA_RTP : URMA_CTP; + *priority = *tp_type == URMA_CTP ? 6 : 0; + return true; +} +urma_tp_type_t GetUrmaTpType() { return g_tp_type; } uint8_t GetUrmaJettyPriority() { return g_jetty_priority; } -bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority) { +bool IsUrmaPrioritySupported(const urma_device_attr_t& attr, int priority, + urma_tp_type_t tp_type) { return priority >= 0 && priority <= URMA_MAX_PRIORITY && - attr.dev_cap.priority_info[priority].tp_type.bs.ctp; + (tp_type == URMA_CTP || tp_type == URMA_RTP) && + (attr.dev_cap.priority_info[priority].tp_type.value & + TpTypeCapability(tp_type).value) != 0; } -static std::string DescribeUrmaCtpSlMap(const urma_device_attr_t& attr) { +static std::string DescribeUrmaSlMap(const urma_device_attr_t& attr, + urma_tp_type_t tp_type) { std::ostringstream os; os << '['; bool first = true; for (int priority = 0; priority <= URMA_MAX_PRIORITY; ++priority) { - if (IsUrmaCtpPrioritySupported(attr, priority)) { + if (IsUrmaPrioritySupported(attr, priority, tp_type)) { if (!first) { os << ','; } @@ -714,9 +748,10 @@ static std::string DescribeUrmaCtpSlMap(const urma_device_attr_t& attr) { } bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, - const UrmaQosOptions& qos, uint8_t default_priority, - uint8_t* priority) { - if (!qos.IsValid()) { + const UrmaQosOptions& qos, urma_tp_type_t tp_type, + uint8_t default_priority, uint8_t* priority) { + if (!qos.IsValid() || !priority || + (tp_type != URMA_CTP && tp_type != URMA_RTP)) { errno = EINVAL; return false; } @@ -726,7 +761,7 @@ bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, } if (qos.priority == -1) { for (int p = 0; p <= URMA_MAX_PRIORITY; ++p) { - if (IsUrmaCtpPrioritySupported(attr, p) && + if (IsUrmaPrioritySupported(attr, p, tp_type) && attr.dev_cap.priority_info[p].SL == static_cast(qos.service_level)) { *priority = static_cast(p); @@ -734,20 +769,21 @@ bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, } } LOG(ERROR) << "URMA SL " << qos.service_level - << " has no CTP priority; supported priority:SL=" - << DescribeUrmaCtpSlMap(attr); - } else if (!IsUrmaCtpPrioritySupported(attr, qos.priority)) { + << " has no " << (tp_type == URMA_CTP ? "CTP" : "RTP") + << " priority; supported priority:SL=" + << DescribeUrmaSlMap(attr, tp_type); + } else if (!IsUrmaPrioritySupported(attr, qos.priority, tp_type)) { LOG(ERROR) << "URMA priority " << qos.priority - << " does not advertise CTP on the selected device; " - "requested SL=" << qos.service_level - << "; supported priority:SL=" << DescribeUrmaCtpSlMap(attr); + << " does not advertise " << (tp_type == URMA_CTP ? "CTP" : "RTP") + << " on the selected device; requested SL=" << qos.service_level + << "; supported priority:SL=" << DescribeUrmaSlMap(attr, tp_type); } else if (qos.service_level != -1 && attr.dev_cap.priority_info[qos.priority].SL != static_cast(qos.service_level)) { LOG(ERROR) << "URMA priority " << qos.priority << " maps to SL " << attr.dev_cap.priority_info[qos.priority].SL << ", requested SL=" << qos.service_level - << "; supported priority:SL=" << DescribeUrmaCtpSlMap(attr); + << "; supported priority:SL=" << DescribeUrmaSlMap(attr, tp_type); } else { *priority = static_cast(qos.priority); return true; @@ -757,6 +793,10 @@ bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, } bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { + if (!priority) { + errno = EINVAL; + return false; + } if (qos.IsValid() && qos.priority == -1 && qos.service_level == -1) { *priority = GetUrmaJettyPriority(); return true; @@ -767,11 +807,13 @@ bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority) { errno = EINVAL; return false; } - return ResolveUrmaJettyPriority(g_device_attr, qos, GetUrmaJettyPriority(), priority); + return ResolveUrmaJettyPriority(g_device_attr, qos, GetUrmaTpType(), + GetUrmaJettyPriority(), priority); } int64_t GetUrmaServiceLevelForPriority(int priority) { - if (!g_context || !IsUrmaCtpPrioritySupported(g_device_attr, priority)) { + if (!g_context || + !IsUrmaPrioritySupported(g_device_attr, priority, GetUrmaTpType())) { return -1; } return g_device_attr.dev_cap.priority_info[priority].SL; diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 274f5a7a0c..1b84787acd 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -81,19 +81,26 @@ bool IsUrmaBondingDevice(); int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, urma_tp_type_t tp_type); -// Return the default CTP Jetty priority selected for the device. +// Select the process-wide RM path and its default priority from device capabilities. +bool SelectUrmaTransportPath(const urma_device_attr_t& attr, + const std::string& requested, + urma_tp_type_t* tp_type, uint8_t* priority); +urma_tp_type_t GetUrmaTpType(); + +// Return the default Jetty priority selected for the device. uint8_t GetUrmaJettyPriority(); -// Check whether an explicit priority advertises the CTP path used by brpc. -bool IsUrmaCtpPrioritySupported(const urma_device_attr_t& attr, int priority); +// Check whether a priority advertises the selected RM path. +bool IsUrmaPrioritySupported(const urma_device_attr_t& attr, int priority, + urma_tp_type_t tp_type); // Resolve a requested priority/SL pair against a device capability snapshot. // Fully automatic options retain default_priority, including legacy fallback. bool ResolveUrmaJettyPriority(const urma_device_attr_t& attr, - const UrmaQosOptions& qos, uint8_t default_priority, - uint8_t* priority); + const UrmaQosOptions& qos, urma_tp_type_t tp_type, + uint8_t default_priority, uint8_t* priority); // Resolve automatic selection or validate against the initialized device. bool ResolveUrmaJettyPriority(const UrmaQosOptions& qos, uint8_t* priority); -// Return the advertised CTP SL, or -1 when no initialized mapping is available. +// Return the advertised SL on the selected path, or -1 when no initialized mapping is available. int64_t GetUrmaServiceLevelForPriority(int priority); // If the URMA environment is available. diff --git a/src/brpc/urma/urma_qos.h b/src/brpc/urma/urma_qos.h index 6c0a1936d2..2d28ff4434 100644 --- a/src/brpc/urma/urma_qos.h +++ b/src/brpc/urma/urma_qos.h @@ -20,7 +20,7 @@ namespace brpc { -// Outbound CTP QoS, resolved through the selected device's priority/SL map. +// Outbound RM QoS, resolved through the selected device's priority/SL map. struct UrmaQosOptions { int priority = -1; // -1 selects from SL or device default; explicit [0, 15]. int service_level = -1; // -1 leaves SL unspecified; explicit range [0, 255]. diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a0fc2d0536..17925787af 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -295,6 +295,12 @@ foreach(BRPC_UT ${BRPC_UNITTESTS}) brpc_test_config ${GPERFTOOLS_LIBRARIES}) add_test(NAME ${BRPC_UT_WE} COMMAND ${BRPC_UT_WE}) + if(WITH_URMA AND BRPC_UT_WE STREQUAL "brpc_urma_qos_unittest") + add_test(NAME brpc_urma_qos_rtp_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp) + add_test(NAME brpc_urma_qos_rtp_polling_unittest + COMMAND ${BRPC_UT_WE} --urma_tp_type=rtp --urma_use_polling=true) + endif() endforeach() if(BUILD_FUZZ_TESTS) diff --git a/test/brpc_rdma_qos_unittest.cpp b/test/brpc_rdma_qos_unittest.cpp new file mode 100644 index 0000000000..06b9778d8c --- /dev/null +++ b/test/brpc_rdma_qos_unittest.cpp @@ -0,0 +1,286 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +#include +#include +#include +#include +#if BRPC_WITH_RDMA +#include +#include +#include +#include +#include +#include "butil/endpoint.h" +#include "butil/fd_guard.h" +#include "butil/time.h" +#include "brpc/acceptor.h" +#include "brpc/channel.h" +#include "brpc/input_messenger.h" +#include "brpc/server.h" +#include "brpc/socket.h" +#include "brpc/rdma_transport.h" +#include "brpc/rdma/rdma_endpoint.h" +#include "brpc/rdma/rdma_handshake.h" +#include "brpc/rdma/rdma_helper.h" +#include "brpc/details/load_balancer_with_naming.h" +#include "butil/files/temp_file.h" + +using namespace brpc; + +namespace brpc { +namespace rdma { +extern butil::atomic g_rdma_available; +extern bool g_skip_rdma_init; +extern int (*IbvModifyQp)(ibv_qp*, ibv_qp_attr*, ibv_qp_attr_mask); +} // namespace rdma +} // namespace brpc + +static butil::EndPoint g_ep; +static const int64_t WAIT_TIMEOUT_US = 5000000; +static const int64_t WAIT_INTERVAL_US = 1000; + +static bool WaitUntil(const std::function& pred, + int64_t timeout_us = WAIT_TIMEOUT_US) { + const int64_t deadline = butil::gettimeofday_us() + timeout_us; + while (!pred()) { + if (butil::gettimeofday_us() >= deadline) { + return false; + } + usleep((useconds_t)WAIT_INTERVAL_US); + } + return true; +} + +static void ConnectToServer(butil::fd_guard* sockfd) { + sockaddr_in addr; + bzero((char*)&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_port = htons(g_ep.port); + addr.sin_addr = g_ep.ip; + sockfd->reset(socket(AF_INET, SOCK_STREAM, 0)); + ASSERT_TRUE(*sockfd >= 0); + timeval tv; + tv.tv_sec = WAIT_TIMEOUT_US / 1000000; + tv.tv_usec = WAIT_TIMEOUT_US % 1000000; + ASSERT_EQ(0, setsockopt(*sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))); + ASSERT_EQ(0, connect(*sockfd, (sockaddr*)&addr, sizeof(addr))); +} + +class RdmaQosTest : public ::testing::Test { +protected: + RdmaQosTest() { + EXPECT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &g_ep)); + EXPECT_EQ(0, _server_list.save(butil::endpoint2str(g_ep).c_str())); + _naming_url = std::string("File://") + _server_list.fname(); + } + + void TearDown() override { StopServer(); } + + void StopServer() { + _server.Stop(0); + _server.Join(); + } + + Socket* GetSocketFromServer(size_t index) { + std::vector sids; + _server._am->ListConnections(&sids); + if (index >= sids.size()) { + return nullptr; + } + SocketUniquePtr socket; + return Socket::Address(sids[index], &socket) == 0 ? socket.get() : nullptr; + } + + Socket* WaitForServerSocket() { + Socket* socket = nullptr; + WaitUntil([this, &socket] { + return (socket = GetSocketFromServer(0)) != nullptr; + }); + return socket; + } + + bool WaitForServerSocketGone() { + return WaitUntil([this] { return GetSocketFromServer(0) == nullptr; }); + } + + butil::TempFile _server_list; + std::string _naming_url; + Server _server; +}; + +static RdmaTransport* RdmaTransportOf(Socket* s) { + return static_cast(s->_transport.get()); +} +static RdmaTransport* RdmaTransportOf(const SocketUniquePtr& s) { + return RdmaTransportOf(s.get()); +} + +TEST_F(RdmaQosTest, qos_rejects_invalid_options_before_connecting) { + for (const auto& values : {std::pair(-1, 0), {16, 0}, {0, -1}, {0, 256}}) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = values.first; + options.rdma_qos.traffic_class = values.second; + Channel channel; + EXPECT_EQ(-1, channel.Init(g_ep, &options)); + ServerOptions server_options; + server_options.socket_mode = SOCKET_MODE_RDMA; + server_options.rdma_qos = options.rdma_qos; + Server server; + EXPECT_EQ(-1, server.Start("127.0.0.1:0", &server_options)); + SocketOptions socket_options; + socket_options.socket_mode = SOCKET_MODE_RDMA; + socket_options.rdma_qos = options.rdma_qos; + SocketId sid; + EXPECT_EQ(-1, Socket::Create(socket_options, &sid)); + EXPECT_EQ(EINVAL, errno); + } +} + +TEST_F(RdmaQosTest, qos_isolates_channels_and_preserves_equal_config_sharing) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + Channel a, b, same, different_tc, default_rdma, tcp, tcp_with_qos; + ASSERT_EQ(0, default_rdma.Init(g_ep, &options)); + ASSERT_EQ(0, tcp.Init(g_ep, nullptr)); + EXPECT_NE(default_rdma._server_id, tcp._server_id); + options.rdma_qos.service_level = 3; + options.rdma_qos.traffic_class = 106; + ASSERT_EQ(0, a.Init(g_ep, &options)); + ASSERT_EQ(0, same.Init(g_ep, &options)); + EXPECT_EQ(a._server_id, same._server_id); + EXPECT_NE(a._server_id, default_rdma._server_id); + options.rdma_qos.service_level = 4; + ASSERT_EQ(0, b.Init(g_ep, &options)); + EXPECT_NE(a._server_id, b._server_id); + options.rdma_qos.service_level = 3; + options.rdma_qos.traffic_class = 104; + ASSERT_EQ(0, different_tc.Init(g_ep, &options)); + EXPECT_NE(a._server_id, different_tc._server_id); + options.socket_mode = SOCKET_MODE_TCP; + ASSERT_EQ(0, tcp_with_qos.Init(g_ep, &options)); + EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(a._server_id, &socket)); + EXPECT_EQ(3, socket->rdma_qos().service_level); + EXPECT_EQ(106, socket->rdma_qos().traffic_class); +} + +TEST_F(RdmaQosTest, qos_follows_pooled_short_and_named_connections) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = 7; + options.rdma_qos.traffic_class = 255; + Channel direct, named; + ASSERT_EQ(0, direct.Init(g_ep, &options)); + ASSERT_EQ(0, named.Init(_naming_url.c_str(), "rr", &options)); + SocketUniquePtr main_socket; + ASSERT_EQ(0, Socket::Address(direct._server_id, &main_socket)); + SocketUniquePtr pooled, short_socket, selected; + ASSERT_EQ(0, main_socket->GetPooledSocket(&pooled)); + ASSERT_EQ(0, main_socket->GetShortSocket(&short_socket)); + LoadBalancer::SelectIn input{}; + LoadBalancer::SelectOut output(&selected); + ASSERT_EQ(0, named._lb->SelectServer(input, &output)); + for (Socket* s : {pooled.get(), short_socket.get(), selected.get()}) { + EXPECT_EQ(7, s->rdma_qos().service_level); + EXPECT_EQ(255, s->rdma_qos().traffic_class); + } + EXPECT_EQ(main_socket->id(), selected->id()); + EXPECT_EQ(0, pooled->ReturnToPool()); + short_socket->SetFailed(); +} + +TEST_F(RdmaQosTest, qos_server_options_follow_accept_and_restart) { + for (int sl : {3, 7}) { + ServerOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = sl; + options.rdma_qos.traffic_class = 106; + ASSERT_EQ(0, _server.Start("127.0.0.1:0", &options)); + g_ep = _server.listen_address(); + butil::fd_guard fd; + ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); + Socket* socket = WaitForServerSocket(); + ASSERT_TRUE(socket != nullptr); + EXPECT_EQ(sl, socket->rdma_qos().service_level); + EXPECT_EQ(106, socket->rdma_qos().traffic_class); + fd.reset(-1); + ASSERT_TRUE(WaitForServerSocketGone()); + StopServer(); + } +} + +static ibv_ah_attr g_qos_rtr_address; +static int g_qos_rtr_calls; +static int CaptureQosModify(ibv_qp*, ibv_qp_attr* attr, ibv_qp_attr_mask mask) { + if (mask & IBV_QP_AV) { + g_qos_rtr_address = attr->ah_attr; + ++g_qos_rtr_calls; + } + return 0; +} + +TEST_F(RdmaQosTest, qos_applies_to_rtr_on_each_qp_bringup) { + for (const auto& values : {std::pair(15, 255), {0, 0}}) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.rdma_qos.service_level = values.first; + options.rdma_qos.traffic_class = values.second; + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + ibv_qp qp{}; + rdma::RdmaResource resource; + resource.qp = &qp; + ep->_resource = &resource; + ep->_rq_size = 0; + rdma::ParsedHello remote{}; + const bool skip = rdma::g_skip_rdma_init; + const auto modify = rdma::IbvModifyQp; + rdma::g_skip_rdma_init = false; + rdma::IbvModifyQp = CaptureQosModify; + g_qos_rtr_calls = 0; + // Reapply attributes on both client and server bringup. + EXPECT_EQ(0, ep->BringUpQp(remote, false)); + EXPECT_EQ(0, ep->BringUpQp(remote, true)); + rdma::g_skip_rdma_init = skip; + rdma::IbvModifyQp = modify; + EXPECT_EQ(2, g_qos_rtr_calls); + EXPECT_EQ(values.first, g_qos_rtr_address.sl); + EXPECT_EQ(values.second, g_qos_rtr_address.grh.traffic_class); + ep->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); + } +} + +#endif // BRPC_WITH_RDMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); +#if BRPC_WITH_RDMA + rdma::g_rdma_available.store(true, butil::memory_order_relaxed); + rdma::g_skip_rdma_init = true; +#endif + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index 1d3ac2c934..6ec54ed90b 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -25,7 +25,6 @@ #include #include #include -#include #include #include "butil/endpoint.h" #include "butil/fd_guard.h" @@ -48,7 +47,6 @@ #include "brpc/rdma/rdma_handshake_constants.h" #include "brpc/rdma/rdma_handshake.pb.h" #include "brpc/rdma/rdma_helper.h" -#include "brpc/details/load_balancer_with_naming.h" #include "echo.pb.h" static const int PORT = 8713; @@ -3318,147 +3316,6 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } -TEST_F(RdmaTest, qos_rejects_invalid_options_before_connecting) { - for (const auto& values : {std::pair(-1, 0), {16, 0}, {0, -1}, {0, 256}}) { - ChannelOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.rdma_qos.service_level = values.first; - options.rdma_qos.traffic_class = values.second; - Channel channel; - EXPECT_EQ(-1, channel.Init(g_ep, &options)); - ServerOptions server_options; - server_options.socket_mode = SOCKET_MODE_RDMA; - server_options.rdma_qos = options.rdma_qos; - Server server; - EXPECT_EQ(-1, server.Start(PORT, &server_options)); - SocketOptions socket_options; - socket_options.socket_mode = SOCKET_MODE_RDMA; - socket_options.rdma_qos = options.rdma_qos; - SocketId sid; - EXPECT_EQ(-1, Socket::Create(socket_options, &sid)); - EXPECT_EQ(EINVAL, errno); - } -} - -TEST_F(RdmaTest, qos_isolates_channels_and_preserves_equal_config_sharing) { - ChannelOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - Channel a, b, same, different_tc, default_rdma, tcp, tcp_with_qos; - ASSERT_EQ(0, default_rdma.Init(g_ep, &options)); - ASSERT_EQ(0, tcp.Init(g_ep, nullptr)); - EXPECT_NE(default_rdma._server_id, tcp._server_id); - options.rdma_qos.service_level = 3; - options.rdma_qos.traffic_class = 106; - ASSERT_EQ(0, a.Init(g_ep, &options)); - ASSERT_EQ(0, same.Init(g_ep, &options)); - EXPECT_EQ(a._server_id, same._server_id); - EXPECT_NE(a._server_id, default_rdma._server_id); - options.rdma_qos.service_level = 4; - ASSERT_EQ(0, b.Init(g_ep, &options)); - EXPECT_NE(a._server_id, b._server_id); - options.rdma_qos.service_level = 3; - options.rdma_qos.traffic_class = 104; - ASSERT_EQ(0, different_tc.Init(g_ep, &options)); - EXPECT_NE(a._server_id, different_tc._server_id); - options.socket_mode = SOCKET_MODE_TCP; - ASSERT_EQ(0, tcp_with_qos.Init(g_ep, &options)); - EXPECT_EQ(tcp._server_id, tcp_with_qos._server_id); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(a._server_id, &socket)); - EXPECT_EQ(3, socket->rdma_qos().service_level); - EXPECT_EQ(106, socket->rdma_qos().traffic_class); -} - -TEST_F(RdmaTest, qos_follows_pooled_short_and_named_connections) { - ChannelOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.rdma_qos.service_level = 7; - options.rdma_qos.traffic_class = 255; - Channel direct, named; - ASSERT_EQ(0, direct.Init(g_ep, &options)); - ASSERT_EQ(0, named.Init(_naming_url.c_str(), "rr", &options)); - SocketUniquePtr main_socket; - ASSERT_EQ(0, Socket::Address(direct._server_id, &main_socket)); - SocketUniquePtr pooled, short_socket, selected; - ASSERT_EQ(0, main_socket->GetPooledSocket(&pooled)); - ASSERT_EQ(0, main_socket->GetShortSocket(&short_socket)); - LoadBalancer::SelectIn input{}; - LoadBalancer::SelectOut output(&selected); - ASSERT_EQ(0, named._lb->SelectServer(input, &output)); - for (Socket* s : {pooled.get(), short_socket.get(), selected.get()}) { - EXPECT_EQ(7, s->rdma_qos().service_level); - EXPECT_EQ(255, s->rdma_qos().traffic_class); - } - EXPECT_EQ(main_socket->id(), selected->id()); - EXPECT_EQ(0, pooled->ReturnToPool()); - short_socket->SetFailed(); -} - -TEST_F(RdmaTest, qos_server_options_follow_accept_and_restart) { - for (int sl : {3, 7}) { - ServerOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.rdma_qos.service_level = sl; - options.rdma_qos.traffic_class = 106; - ASSERT_EQ(0, _server.Start(PORT, &options)); - butil::fd_guard fd; - ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); - Socket* socket = WaitForServerSocket(); - ASSERT_TRUE(socket != nullptr); - EXPECT_EQ(sl, socket->rdma_qos().service_level); - EXPECT_EQ(106, socket->rdma_qos().traffic_class); - fd.reset(-1); - ASSERT_TRUE(WaitForServerSocketGone()); - StopServer(); - } -} - -static ibv_ah_attr g_qos_rtr_address; -static int g_qos_rtr_calls; -static int CaptureQosModify(ibv_qp*, ibv_qp_attr* attr, ibv_qp_attr_mask mask) { - if (mask & IBV_QP_AV) { - g_qos_rtr_address = attr->ah_attr; - ++g_qos_rtr_calls; - } - return 0; -} - -TEST_F(RdmaTest, qos_applies_to_rtr_on_each_qp_bringup) { - for (const auto& values : {std::pair(15, 255), {0, 0}}) { - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.rdma_qos.service_level = values.first; - options.rdma_qos.traffic_class = values.second; - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - ibv_qp qp{}; - rdma::RdmaResource resource; - resource.qp = &qp; - ep->_resource = &resource; - ep->_rq_size = 0; - rdma::ParsedHello remote{}; - const bool skip = rdma::g_skip_rdma_init; - const auto modify = rdma::IbvModifyQp; - rdma::g_skip_rdma_init = false; - rdma::IbvModifyQp = CaptureQosModify; - g_qos_rtr_calls = 0; - // Reapply attributes on both client and server bringup. - EXPECT_EQ(0, ep->BringUpQp(remote, false)); - EXPECT_EQ(0, ep->BringUpQp(remote, true)); - rdma::g_skip_rdma_init = skip; - rdma::IbvModifyQp = modify; - EXPECT_EQ(2, g_qos_rtr_calls); - EXPECT_EQ(values.first, g_qos_rtr_address.sl); - EXPECT_EQ(values.second, g_qos_rtr_address.grh.traffic_class); - ep->_resource = nullptr; - resource.qp = nullptr; - socket->SetFailed(); - } -} - #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { diff --git a/test/brpc_urma_qos_unittest.cpp b/test/brpc_urma_qos_unittest.cpp index bb04a9bee3..5770416c6c 100644 --- a/test/brpc_urma_qos_unittest.cpp +++ b/test/brpc_urma_qos_unittest.cpp @@ -31,6 +31,7 @@ #include "brpc/load_balancer.h" #include "brpc/urma/urma_endpoint.h" #include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_handshake.h" #include "brpc/urma_transport.h" #include "echo.pb.h" @@ -39,6 +40,7 @@ using namespace brpc; namespace brpc { namespace urma { DECLARE_int32(urma_client_handshake_version); +DECLARE_string(urma_tp_type); extern bool g_skip_urma_init; DECLARE_int32(urma_buffer_count); DECLARE_int32(urma_prepared_jetty_cnt); @@ -52,6 +54,14 @@ class UrmaQosTestPeer { static ssize_t Complete(UrmaEndpoint* ep, const urma_cr_t& cr) { return ep->HandleCompletion(cr); } + static int Import(UrmaEndpoint* ep, const ParsedHello& hello) { + return ep->ImportPeer(hello); + } + static ParsedHello Hello(UrmaEndpoint* ep) { + ParsedHello hello; + ep->MakeLocalParsedHello(&hello); + return hello; + } }; } // namespace urma @@ -63,10 +73,10 @@ TEST(UrmaHelperTest, explicit_priority_requires_ctp_capability) { attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; attr.dev_cap.priority_info[15].tp_type.bs.rtp = 1; attr.dev_cap.priority_info[3].tp_type.bs.rtp = 1; - EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 0)); - EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 15)); + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 0, URMA_CTP)); + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 15, URMA_CTP)); for (int priority : {-2, -1, 3, 4, 16}) { - EXPECT_FALSE(urma::IsUrmaCtpPrioritySupported(attr, priority)); + EXPECT_FALSE(urma::IsUrmaPrioritySupported(attr, priority, URMA_CTP)); } } @@ -78,7 +88,7 @@ TEST(UrmaHelperTest, automatic_priority_accepts_combined_capabilities) { attr.dev_cap.priority_info[15].tp_type.bs.ctp = 1; EXPECT_EQ(3, urma::FindUrmaPriorityForTpType(attr, URMA_RTP)); EXPECT_EQ(10, urma::FindUrmaPriorityForTpType(attr, URMA_CTP)); - EXPECT_TRUE(urma::IsUrmaCtpPrioritySupported(attr, 10)); + EXPECT_TRUE(urma::IsUrmaPrioritySupported(attr, 10, URMA_CTP)); EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType( attr, static_cast(-1))); @@ -102,22 +112,22 @@ TEST(UrmaHelperTest, service_level_selects_ctp_priority_from_mapping) { UrmaQosOptions qos; uint8_t priority = 0; qos.service_level = 6; - ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(10, priority); qos.service_level = 7; - ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(6, priority); qos.priority = 6; - ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(6, priority); attr.dev_cap.priority_info[15].SL = 255; qos.priority = -1; qos.service_level = 255; - ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(15, priority); qos.service_level = -1; attr = urma_device_attr_t{}; - ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(6, priority); } @@ -133,13 +143,13 @@ TEST(UrmaHelperTest, service_level_rejects_unsupported_and_conflicting_mapping) uint8_t priority = 255; for (int sl : {-2, 0, 3, 6, 255, 256}) { qos.service_level = sl; - EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(EINVAL, errno); EXPECT_EQ(255, priority); } qos.priority = 6; qos.service_level = 4; - EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority)); + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); EXPECT_EQ(EINVAL, errno); EXPECT_EQ(255, priority); #if !BRPC_WITH_GLOG @@ -147,7 +157,7 @@ TEST(UrmaHelperTest, service_level_rejects_unsupported_and_conflicting_mapping) qos.service_level = 3; logging::StringSink log; logging::LogSink* old_sink = logging::SetLogSink(&log); - const bool resolved = urma::ResolveUrmaJettyPriority(attr, qos, 6, &priority); + const bool resolved = urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority); logging::SetLogSink(old_sink); EXPECT_FALSE(resolved); EXPECT_NE(std::string::npos, log.find("SL 3 has no CTP priority")) << log; @@ -155,6 +165,121 @@ TEST(UrmaHelperTest, service_level_rejects_unsupported_and_conflicting_mapping) #endif } +TEST(UrmaPathQosTest, automatic_and_explicit_paths_use_device_capabilities) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + urma_tp_type_t path = URMA_CTP; + uint8_t priority = 15; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].tp_type.bs.rtp = 1; + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); + EXPECT_EQ(URMA_CTP, path); + EXPECT_EQ(6, priority); + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); + EXPECT_EQ(URMA_RTP, path); + EXPECT_EQ(0, priority); +} + +TEST(UrmaPathQosTest, unsupported_paths_preserve_outputs) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const char* request : {"ctp", "utp", "", "typo"}) { + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, request, &path, &priority)); + EXPECT_EQ(URMA_UTP, path); + EXPECT_EQ(15, priority); + } + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", nullptr, &priority)); + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, nullptr)); + attr.dev_cap.priority_info[0].tp_type.value = 0; + attr.dev_cap.priority_info[4].tp_type.bs.utp = 1; + EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, &priority)); +} + +TEST(UrmaPathQosTest, empty_capability_table_uses_compatibility_defaults) { + urma_device_attr_t attr{}; + urma_tp_type_t path = URMA_UTP; + uint8_t priority = 15; + for (const char* request : {"auto", "ctp", "rtp"}) { + ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, request, &path, &priority)); + const bool rtp = std::string(request) == "rtp"; + EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); + EXPECT_EQ(rtp ? 0 : 6, priority); + UrmaQosOptions qos; + uint8_t resolved = 15; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, path, priority, &resolved)); + EXPECT_EQ(priority, resolved); + qos.service_level = 2; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, path, priority, &resolved)); + } +} + +TEST(UrmaPathQosTest, service_level_resolution_is_scoped_to_the_selected_path) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + attr.dev_cap.priority_info[4].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[4].SL = 4; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 2; + attr.dev_cap.priority_info[10].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[10].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[10].SL = 4; + UrmaQosOptions qos; + qos.service_level = 2; + uint8_t priority = 15; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(0, priority); + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_EQ(6, priority); + qos.service_level = 4; + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(4, priority); + qos.priority = 10; + for (auto path : {URMA_RTP, URMA_CTP}) { + ASSERT_TRUE(urma::ResolveUrmaJettyPriority(attr, qos, path, 0, &priority)); + EXPECT_EQ(10, priority); + } +} + +TEST(UrmaPathQosTest, rejects_cross_path_priorities_and_conflicting_sl) { + urma_device_attr_t attr{}; + attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; + attr.dev_cap.priority_info[0].SL = 2; + attr.dev_cap.priority_info[6].tp_type.bs.ctp = 1; + attr.dev_cap.priority_info[6].SL = 6; + UrmaQosOptions qos; + uint8_t priority = 15; + qos.service_level = 6; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + qos.priority = 6; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + qos.priority = 0; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_CTP, 6, &priority)); + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, &priority)); + EXPECT_EQ(15, priority); + EXPECT_EQ(EINVAL, errno); +} + +TEST(UrmaPathQosTest, rejects_invalid_resolution_arguments) { + urma_device_attr_t attr{}; + UrmaQosOptions qos; + uint8_t priority = 15; + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, URMA_RTP, 0, nullptr)); + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(qos, nullptr)); + for (auto path : {URMA_UTP, static_cast(255)}) { + EXPECT_FALSE(urma::ResolveUrmaJettyPriority(attr, qos, path, 0, &priority)); + EXPECT_FALSE(urma::IsUrmaPrioritySupported(attr, 0, path)); + } + EXPECT_EQ(15, priority); + EXPECT_EQ(EINVAL, errno); +} + class UrmaQosTest : public ::testing::Test { protected: static void SetUpTestSuite() { @@ -193,10 +318,51 @@ class UrmaQosTest : public ::testing::Test { }; bool UrmaQosTest::_mock = false; +TEST_F(UrmaQosTest, service_level_requires_capability_on_the_selected_path) { + ChannelOptions options; + options.socket_mode = SOCKET_MODE_URMA; + options.urma_qos.service_level = 3; + Channel channel; + if (urma::GetUrmaTpType() == URMA_CTP) { + EXPECT_EQ(-1, channel.Init("127.0.0.1:8123", &options)); + } else { + ASSERT_EQ(0, channel.Init("127.0.0.1:8123", &options)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(channel._server_id, &socket)); + ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(Endpoint(socket))); + EXPECT_EQ(3, urma::UrmaQosTestPeer::Priority(Endpoint(socket))); + EXPECT_EQ(3, urma::GetUrmaServiceLevelForPriority(3)); + } +} + +TEST_F(UrmaQosTest, advertises_selected_path_and_rejects_mismatch_before_import) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_URMA; + SocketId id; + ASSERT_EQ(0, Socket::Create(options, &id)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(id, &socket)); + auto* endpoint = Endpoint(socket); + auto hello = urma::UrmaQosTestPeer::Hello(endpoint); + EXPECT_EQ(static_cast(urma::GetUrmaTpType()), hello.tp_type); + for (auto path : {URMA_CTP, URMA_RTP, URMA_UTP, static_cast(255)}) { + if (path == urma::GetUrmaTpType()) { + continue; + } + hello.tp_type = static_cast(path); + EXPECT_EQ(-1, urma::UrmaQosTestPeer::Import(endpoint, hello)); + EXPECT_EQ(EPROTO, errno); + } + socket->SetFailed(); +} + TEST_F(UrmaQosTest, rejects_invalid_and_unsupported_priority) { butil::EndPoint address; ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &address)); - for (int priority : {-2, 16, 3, 9}) { + for (int priority : {-2, 16, 3, 4, 9}) { + if (priority == 3 && urma::GetUrmaTpType() == URMA_RTP) { + continue; + } ChannelOptions co; co.socket_mode = SOCKET_MODE_URMA; co.urma_qos.priority = priority; @@ -232,7 +398,11 @@ TEST_F(UrmaQosTest, rejects_invalid_and_unsupported_priority) { TEST_F(UrmaQosTest, rejects_invalid_unsupported_and_conflicting_service_level) { struct RequestedQos { int priority; int sl; }; for (const auto& requested : {RequestedQos{-1, -2}, {-1, 256}, {-1, 0}, - {-1, 3}, {-1, 6}, {-1, 255}, {6, 4}, {3, 3}}) { + {-1, 3}, {-1, 6}, {-1, 255}, {6, 4}, {3, 3}, {4, 3}}) { + if (requested.sl == 3 && requested.priority != 4 && + urma::GetUrmaTpType() == URMA_RTP) { + continue; + } ChannelOptions co; co.socket_mode = SOCKET_MODE_URMA; co.urma_qos.priority = requested.priority; @@ -339,7 +509,7 @@ TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { // SL 4 resolves to priority 10 and must leave the default Jettys available. struct RequestedQos { int priority; int sl; int effective; }; for (const auto& qos : {RequestedQos{-1, 4, 10}, {-1, 2, 6}, {6, 2, 6}, - {10, -1, 10}, {-1, -1, 6}, {10, 4, 10}}) { + {10, -1, 10}, {-1, -1, urma::GetUrmaJettyPriority()}, {10, 4, 10}}) { SocketOptions options; options.socket_mode = SOCKET_MODE_URMA; options.urma_qos.priority = qos.priority; @@ -357,7 +527,8 @@ TEST_F(UrmaQosTest, prepared_jettys_are_used_only_for_matching_priority) { EXPECT_NE(std::string::npos, debug.str().find( "urma_sl=" + std::to_string(qos.sl))); EXPECT_NE(std::string::npos, debug.str().find( - "urma_effective_sl=" + std::to_string(qos.effective == 6 ? 2 : 4))); + "urma_effective_sl=" + std::to_string( + urma::GetUrmaServiceLevelForPriority(qos.effective)))); socket->SetFailed(); } } @@ -375,7 +546,7 @@ TEST_F(UrmaQosTest, access_error_preserves_priority_and_reports_direction) { auto* ep = Endpoint(socket); ASSERT_EQ(0, urma::UrmaQosTestPeer::Allocate(ep)); const int effective = - priority == -1 ? urma::GetUrmaJettyPriority() : priority; + priority == -1 ? 6 : priority; for (bool receive : {false, true}) { urma_cr_t cr{}; cr.status = URMA_CR_LOC_ACCESS_ERR; From 8ef57ed95d9099cae84801c7644a9980a371c764 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sat, 31 Oct 2026 04:01:33 -0700 Subject: [PATCH 22/29] Classify RDMA transport failures and report receive posting errors --- src/brpc/rdma/rdma_endpoint.cpp | 9 +- src/brpc/rdma/rdma_helper.cpp | 24 ++- src/brpc/rdma/rdma_helper.h | 3 + test/brpc_rdma_error_recovery_unittest.cpp | 220 +++++++++++++++++++++ 4 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 test/brpc_rdma_error_recovery_unittest.cpp diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 3d78e47a75..cb0fcc9d01 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -1189,6 +1189,8 @@ int RdmaEndpoint::DoPostRecv(void* block, size_t block_size) { int err = ibv_post_recv(_resource->qp, &wr, &bad); if (err != 0) { LOG(WARNING) << "Fail to ibv_post_recv: " << berror(err); + ReportFailure(RdmaFailureReason::RECV_POST); + errno = err; return -1; } return 0; @@ -1894,12 +1896,7 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { - if (IsRdmaPathFailure(wc[i].status)) { - ep->ReportFailure(wc[i].status == IBV_WC_RNR_RETRY_EXC_ERR ? - RdmaFailureReason::RNR : RdmaFailureReason::RETRY_EXHAUSTED); - } else if (wc[i].status != IBV_WC_WR_FLUSH_ERR) { - ep->ReportFailure(RdmaFailureReason::ACCESS_ERROR); - } + ep->ReportFailure(ClassifyRdmaCompletion(wc[i].status)); PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); s->SetFailed(ERDMA, "RDMA completion error(%d) from %s: %s", diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index 1ff7884acd..dc0e622a75 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -707,8 +707,28 @@ bool IsRdmaAvailable() { } bool IsRdmaPathFailure(ibv_wc_status status) { - return status == IBV_WC_RETRY_EXC_ERR || - status == IBV_WC_RNR_RETRY_EXC_ERR; + const auto reason = ClassifyRdmaCompletion(status); + return reason != RdmaFailureReason::NONE && + reason != RdmaFailureReason::ACCESS_ERROR; +} + +RdmaFailureReason ClassifyRdmaCompletion(ibv_wc_status status) { + switch (status) { + case IBV_WC_SUCCESS: + case IBV_WC_WR_FLUSH_ERR: + return RdmaFailureReason::NONE; + case IBV_WC_RETRY_EXC_ERR: + return RdmaFailureReason::RETRY_EXHAUSTED; + case IBV_WC_RNR_RETRY_EXC_ERR: + return RdmaFailureReason::RNR; + case IBV_WC_LOC_PROT_ERR: + case IBV_WC_LOC_ACCESS_ERR: + case IBV_WC_REM_ACCESS_ERR: + case IBV_WC_MW_BIND_ERR: + return RdmaFailureReason::ACCESS_ERROR; + default: + return RdmaFailureReason::TRANSPORT_ERROR; + } } void GlobalDisableRdma() { diff --git a/src/brpc/rdma/rdma_helper.h b/src/brpc/rdma/rdma_helper.h index d3a840cf1d..5f49f5779f 100644 --- a/src/brpc/rdma/rdma_helper.h +++ b/src/brpc/rdma/rdma_helper.h @@ -29,6 +29,8 @@ namespace brpc { namespace rdma { +enum class RdmaFailureReason; + // Initialize RDMA environment // Exit if failed void GlobalRdmaInitializeOrDie(); @@ -91,6 +93,7 @@ void GlobalDisableRdma(); // Completion failures that indicate an unusable path or exhausted receive credits. bool IsRdmaPathFailure(ibv_wc_status status); +RdmaFailureReason ClassifyRdmaCompletion(ibv_wc_status status); // If the given protocol supported by RDMA bool SupportedByRdma(std::string protocol); diff --git a/test/brpc_rdma_error_recovery_unittest.cpp b/test/brpc_rdma_error_recovery_unittest.cpp new file mode 100644 index 0000000000..1eaaaf5d71 --- /dev/null +++ b/test/brpc_rdma_error_recovery_unittest.cpp @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_RDMA +#include "brpc/global.h" +#include "brpc/input_messenger.h" +#include "brpc/rdma/rdma_endpoint.h" +#include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/rdma_transport.h" +#include "brpc/socket.h" + +namespace brpc { +namespace rdma { +extern bool g_skip_rdma_init; +extern butil::atomic g_rdma_available; +} // namespace rdma +} // namespace brpc + +namespace { +using namespace brpc; + +struct CompletionSource { + ibv_wc_status status; + bool delivered = false; +}; + +int PollError(ibv_cq* cq, int, ibv_wc* wc) { + auto* source = static_cast(cq->cq_context); + if (source->delivered) { + return 0; + } + source->delivered = true; + std::memset(wc, 0, sizeof(*wc)); + wc->status = source->status; + return 1; +} + +int FailRecvPost(ibv_qp*, ibv_recv_wr*, ibv_recv_wr**) { + return EIO; +} + +class RdmaErrorRecoveryTest : public testing::Test { +protected: + void SetUp() override { + GlobalInitializeOrDie(); + _saved_mode = rdma::GetRdmaRecoveryMode(); + ResetPolicy(); + } + + void TearDown() override { + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(_saved_mode); + } + + void ResetPolicy() { + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + } + + void CreateSocket(SocketUniquePtr* socket) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &options.remote_side)); + options.user = get_client_side_messenger(); + SocketId id; + ASSERT_EQ(0, Socket::Create(options, &id)); + ASSERT_EQ(0, Socket::Address(id, socket)); + } + + static rdma::RdmaEndpoint* Endpoint(const SocketUniquePtr& socket) { + return static_cast(socket->_transport.get())->GetRdmaEp(); + } + + void CheckCompletion(ibv_wc_status status, rdma::RdmaFailureReason reason, + bool cooldown) { + ResetPolicy(); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + auto* endpoint = Endpoint(socket); + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + ASSERT_TRUE(policy.AllowNewConnection(_peer, &endpoint->_recovery_epoch)); + const auto before = policy.GetStatus(); + SocketOptions carrier_options; + carrier_options.user = endpoint; + SocketId carrier_id; + ASSERT_EQ(0, Socket::Create(carrier_options, &carrier_id)); + SocketUniquePtr carrier; + ASSERT_EQ(0, Socket::Address(carrier_id, &carrier)); + + CompletionSource source{status}; + ibv_context context{}; + context.ops.poll_cq = PollError; + ibv_cq cq{}; + cq.context = &context; + cq.cq_context = &source; + rdma::RdmaResource resource; + resource.polling_cq = &cq; + endpoint->_resource = &resource; + endpoint->_cq_sid = carrier_id; + endpoint->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + rdma::FLAGS_rdma_use_polling = true; + + rdma::RdmaEndpoint::PollCq(carrier.get()); + EXPECT_TRUE(source.delivered); + EXPECT_TRUE(socket->Failed()); + EXPECT_EQ(reason, endpoint->_runtime_failure.load()); + EXPECT_EQ(!cooldown, policy.AllowNewConnection(_peer)); + EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); + const auto after = policy.GetStatus(); + for (size_t i = 1; i < static_cast(rdma::RdmaFailureReason::COUNT); ++i) { + EXPECT_EQ(before.failures[i] + (i == static_cast(reason)), + after.failures[i]); + } + if (reason != rdma::RdmaFailureReason::NONE) { + endpoint->ReportFailure(rdma::RdmaFailureReason::CQ_ERROR); + EXPECT_EQ(after.failures[static_cast(rdma::RdmaFailureReason::CQ_ERROR)], + policy.GetStatus().failures[ + static_cast(rdma::RdmaFailureReason::CQ_ERROR)]); + } + + endpoint->_resource = nullptr; + endpoint->_cq_sid = INVALID_SOCKET_ID; + resource.polling_cq = nullptr; + carrier->_user = nullptr; + carrier->SetFailed(); + } + + const std::string _peer = "127.0.0.1:8123"; + GFLAGS_NAMESPACE::FlagSaver _flags; + rdma::RdmaRecoveryMode _saved_mode; +}; + +TEST_F(RdmaErrorRecoveryTest, fatal_timeout_and_general_completions_cool_peer) { + for (auto status : {IBV_WC_FATAL_ERR, IBV_WC_RESP_TIMEOUT_ERR, IBV_WC_GENERAL_ERR}) { + ASSERT_NO_FATAL_FAILURE(CheckCompletion( + status, rdma::RdmaFailureReason::TRANSPORT_ERROR, true)); + } +} + +TEST_F(RdmaErrorRecoveryTest, retry_and_rnr_completions_cool_peer) { + ASSERT_NO_FATAL_FAILURE(CheckCompletion( + IBV_WC_RETRY_EXC_ERR, rdma::RdmaFailureReason::RETRY_EXHAUSTED, true)); + ASSERT_NO_FATAL_FAILURE(CheckCompletion( + IBV_WC_RNR_RETRY_EXC_ERR, rdma::RdmaFailureReason::RNR, true)); +} + +TEST_F(RdmaErrorRecoveryTest, access_and_flush_completions_preserve_admission) { + for (auto status : {IBV_WC_LOC_PROT_ERR, IBV_WC_LOC_ACCESS_ERR, + IBV_WC_REM_ACCESS_ERR, IBV_WC_MW_BIND_ERR}) { + ASSERT_NO_FATAL_FAILURE(CheckCompletion( + status, rdma::RdmaFailureReason::ACCESS_ERROR, false)); + } + ASSERT_NO_FATAL_FAILURE(CheckCompletion( + IBV_WC_WR_FLUSH_ERR, rdma::RdmaFailureReason::NONE, false)); +} + +TEST_F(RdmaErrorRecoveryTest, receive_post_error_preserves_errno_and_reports_once) { + for (bool established : {false, true}) { + ResetPolicy(); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + auto* endpoint = Endpoint(socket); + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + ASSERT_TRUE(policy.AllowNewConnection(_peer, &endpoint->_recovery_epoch)); + const auto before = policy.GetStatus(); + ibv_context context{}; + context.ops.post_recv = FailRecvPost; + ibv_qp qp{}; + qp.context = &context; + rdma::RdmaResource resource; + resource.qp = &qp; + endpoint->_resource = &resource; + endpoint->_state.store(established ? rdma::RdmaEndpoint::ESTABLISHED : + rdma::RdmaEndpoint::C_ALLOC_QPCQ); + char data[32]{}; + for (int attempt = 0; attempt < 2; ++attempt) { + errno = EALREADY; + EXPECT_EQ(-1, endpoint->DoPostRecv(data, sizeof(data))); + EXPECT_EQ(EIO, errno); + } + EXPECT_EQ(!established, policy.AllowNewConnection(_peer)); + const size_t index = static_cast(rdma::RdmaFailureReason::RECV_POST); + EXPECT_EQ(before.failures[index] + established, policy.GetStatus().failures[index]); + endpoint->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); + } +} +} // namespace +#endif // BRPC_WITH_RDMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); +#if BRPC_WITH_RDMA + brpc::rdma::g_rdma_available.store(true, butil::memory_order_relaxed); + brpc::rdma::g_skip_rdma_init = true; +#endif + return RUN_ALL_TESTS(); +} From 9e6bd983eeda2269aad9c6d01a3a8dddad330f58 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Sun, 1 Nov 2026 22:58:47 -0800 Subject: [PATCH 23/29] Keep runtime recovery scoped to URMA --- docs/cn/rdma.md | 5 - docs/cn/rdma_runtime_recovery.md | 78 --- docs/cn/urma.md | 2 +- docs/en/urma.md | 4 +- example/BUILD.bazel | 10 +- example/rdma_performance/CMakeLists.txt | 4 - example/rdma_performance/Makefile | 11 +- example/rdma_performance/RECOVERY.md | 84 --- example/urma_performance/CMakeLists.txt | 2 +- .../recovery_client.cpp | 42 +- .../run_recovery_injection.py | 2 +- .../test_run_recovery_injection.py | 4 +- src/brpc/rdma/rdma_endpoint.cpp | 99 +--- src/brpc/rdma/rdma_endpoint.h | 11 - src/brpc/rdma/rdma_helper.cpp | 40 +- src/brpc/rdma/rdma_helper.h | 6 - src/brpc/rdma/rdma_recovery_policy.h | 94 ---- src/brpc/urma/urma_endpoint.cpp | 2 +- .../urma_recovery_policy.cpp} | 142 ++--- src/brpc/urma/urma_recovery_policy.h | 58 +- test/brpc_rdma_error_recovery_unittest.cpp | 220 -------- test/brpc_rdma_runtime_unittest.cpp | 510 ------------------ ...st.cpp => brpc_urma_recovery_unittest.cpp} | 97 ++-- test/brpc_urma_runtime_unittest.cpp | 7 +- 24 files changed, 172 insertions(+), 1362 deletions(-) delete mode 100644 docs/cn/rdma_runtime_recovery.md delete mode 100644 example/rdma_performance/RECOVERY.md rename example/{rdma_performance => urma_performance}/recovery_client.cpp (77%) delete mode 100644 src/brpc/rdma/rdma_recovery_policy.h rename src/brpc/{rdma/rdma_recovery_policy.cpp => urma/urma_recovery_policy.cpp} (54%) delete mode 100644 test/brpc_rdma_error_recovery_unittest.cpp delete mode 100644 test/brpc_rdma_runtime_unittest.cpp rename test/{brpc_rdma_recovery_unittest.cpp => brpc_urma_recovery_unittest.cpp} (63%) diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index 7dd2689156..fd70686bb1 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,8 +1,3 @@ -# 运行期故障恢复 - -模式切换、端口与对端冷却及 TCP 回落见 -[RDMA 与 URMA 运行期故障恢复](rdma_runtime_recovery.md)。 - # 编译 由于RDMA对驱动与硬件有要求,目前仅支持在Linux系统编译并运行RDMA功能。 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md deleted file mode 100644 index 0a2f71e313..0000000000 --- a/docs/cn/rdma_runtime_recovery.md +++ /dev/null @@ -1,78 +0,0 @@ - - -# RDMA 与 URMA 运行期故障恢复 - -运行期故障后,恢复策略控制后续连接的传输选择: - -故障上报 → 对端或端口冷却 → 新连接使用 TCP → 冷却结束后恢复准入。 - -## 模式 - -| 模式 | 新连接行为 | -| --- | --- | -| LEGACY(默认) | 沿用原有握手和错误处理 | -| AUTO | 按端口和对端状态准入,冷却期间选择 TCP | -| TCP_ONLY | 直接选择 TCP | - -模式切换作用于新连接,存量连接继续使用已协商的传输。 -当前 RPC 沿用 Controller 的重试次数和 deadline。 - -## 冷却与错误处理 - -默认冷却 5 秒,使用单调时钟。客户端按服务端 IP 和端口记录对端状态, -缓存上限为 1024 项,满时淘汰最早到期项。RDMA 和 URMA 独立维护状态与指标, -模式切换递增 epoch,过滤旧连接的故障上报。 - -| 事件 | 处理 | -| --- | --- | -| RDMA PORT_ERR | 暂停当前端口的新 RDMA 连接,记录端口冷却 | -| RDMA PORT_ACTIVE | 更新端口状态,冷却结束后恢复准入 | -| 重试耗尽、RNR、QP/CQ/JFC 错误、WR 提交失败 | 上报故障,AUTO 下冷却客户端对端 | -| 访问错误 | 单独计数 | -| WR flush | 沿用已记录的根因 | -| SQ 或接收 credit 耗尽 | 返回 EAGAIN,等待 completion 或 credit | - -同一连接只记录首个运行期故障。QP/CQ 异步事件核对资源归属,故障资源释放。 -URMA 握手支持 v2/v3;短连接关闭后回收队列和接收缓冲,TCP 解析与关闭检查串行执行。 - -## 配置 - -```cpp -#include "brpc/rdma/rdma_recovery_policy.h" -#include "brpc/urma/urma_recovery_policy.h" - -brpc::rdma::SetRdmaRecoveryCooldownMs(5000); -brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::AUTO); - -brpc::urma::SetUrmaRecoveryCooldownMs(5000); -brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); -``` - -运行时调用对应 setter 切换 `LEGACY`、`AUTO` 或 `TCP_ONLY`。 -URMA 使用 CTP/RTP 路径,设备选择见 [URMA 配置](urma.md#传输路径选择)。 - -## 指标与验证 - -`GetRdmaRecoveryStatus()`、`GetUrmaRecoveryStatus()` 及 `/vars/rdma_recovery_*`、 -`/vars/urma_recovery_*` 提供模式、epoch、分原因故障计数、冷却状态、 -传输协商数和 TCP 准入数。TCP 准入按连接累计。 - -- [RDMA 双机测试](../../example/rdma_performance/RECOVERY.md):模式切换、端口和对端故障。 -- [URMA 定向注入](urma.md#定向发送失败测试):一次 SEND 提交失败、TCP 冷却和 URMA 恢复。 diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 48c822c622..107809746e 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -205,7 +205,7 @@ brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); 默认冷却 5 秒,故障来源包括重试耗尽、RNR、JFC 错误和 WR 提交失败。 同一连接记录首个运行期故障;访问错误单独计数,窗口耗尽保持 EAGAIN。 -URMA 与 RDMA 独立维护模式、epoch 和冷却缓存,模式切换过滤旧连接上报。 +恢复策略维护进程级模式、epoch 和对端冷却缓存,模式切换过滤旧连接上报。 `GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态及协商数; `/vars/urma_recovery_tcp_fallback_admissions` 按连接累计 TCP 准入。 diff --git a/docs/en/urma.md b/docs/en/urma.md index 03cfc19098..d81f68d2fd 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -218,8 +218,8 @@ brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); The default cooldown is five seconds. Sources include retry exhaustion, RNR, JFC errors and WR submission failures. Each connection records its first runtime failure; access errors have a separate counter, and exhausted windows return EAGAIN. -URMA and RDMA keep separate modes, epochs and cooldown caches. Mode changes filter -reports from older connections. +The recovery policy maintains process-level modes, epochs and peer cooldowns. +Mode changes filter reports from older connections. `GetUrmaRecoveryStatus()` and `/vars/urma_recovery_status` expose failure reasons, cooldown state and negotiation counts. `/vars/urma_recovery_tcp_fallback_admissions` diff --git a/example/BUILD.bazel b/example/BUILD.bazel index 9049cf07d7..05eda2080b 100644 --- a/example/BUILD.bazel +++ b/example/BUILD.bazel @@ -123,14 +123,6 @@ cc_binary( ], ) -cc_binary( - name = "rdma_recovery_client", - srcs = ["rdma_performance/recovery_client.cpp"], - includes = ["rdma_performance"], - copts = COPTS + ["-DBRPC_WITH_RDMA=1"], - deps = [":cc_rdma_performance_proto", "//:brpc"], -) - cc_binary( name = "ubring_performance_server", srcs = [ @@ -170,4 +162,4 @@ cc_binary( deps = [ "//:brpc", ], -) +) \ No newline at end of file diff --git a/example/rdma_performance/CMakeLists.txt b/example/rdma_performance/CMakeLists.txt index 7e0e70f724..213cff0f24 100644 --- a/example/rdma_performance/CMakeLists.txt +++ b/example/rdma_performance/CMakeLists.txt @@ -40,7 +40,3 @@ brpc_example_configure_target(server) target_link_libraries(client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) target_link_libraries(server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) - -add_executable(recovery_client recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) -brpc_example_configure_target(recovery_client) -target_link_libraries(recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/rdma_performance/Makefile b/example/rdma_performance/Makefile index 58b40acf8b..c112f1c330 100644 --- a/example/rdma_performance/Makefile +++ b/example/rdma_performance/Makefile @@ -59,7 +59,7 @@ endif .PHONY:all ifdef WITH_RDMA -all: client server recovery_client +all: client server else all: endif @@ -67,7 +67,7 @@ endif .PHONY:clean clean: @echo "> Cleaning" - rm -rf client server recovery_client recovery_client.o $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) + rm -rf client server $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) client:$(PROTO_OBJS) $(CLIENT_OBJS) @echo "> Linking $@" @@ -85,13 +85,6 @@ else $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ endif -recovery_client:$(PROTO_OBJS) recovery_client.o -ifneq ("$(LINK_SO)", "") - $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ -else - $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ -endif - %.pb.cc %.pb.h:%.proto @echo "> Generating $@" $(PROTOC) --cpp_out=. --proto_path=. $(PROTOC_EXTRA_ARGS) $< diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md deleted file mode 100644 index cf3a18497e..0000000000 --- a/example/rdma_performance/RECOVERY.md +++ /dev/null @@ -1,84 +0,0 @@ - - -# RDMA 双机故障恢复测试 - -两台 Linux 主机运行服务端和 recovery_client,TCP 控制地址保持可达。 -记录设备、端口、GID、驱动和固件版本。 - -## 编译 - -在仓库根目录执行: - -```bash -cmake -S . -B build -DWITH_RDMA=ON -DBUILD_SHARED_LIBS=OFF -cmake --build build -j8 -cmake --install build --prefix "$PWD/build/install" -cmake -S example/rdma_performance -B build/recovery -DCMAKE_PREFIX_PATH="$PWD/build/install" -cmake --build build/recovery --target server recovery_client -j8 -``` - -## 运行与切换 - -服务端: - -```bash -./build/recovery/server --port=8002 --use_rdma=true --rdma_device=mlx5_0 --rdma_gid_index=3 -``` - -客户端,将 B_TCP_IP 替换为服务端地址: - -```bash -printf 'auto\n' > /tmp/brpc-rdma-mode -./build/recovery/recovery_client --server=B_TCP_IP:8002 --seconds=180 \ - --mode_file=/tmp/brpc-rdma-mode --connection_type=short \ - --rdma_device=mlx5_0 --rdma_gid_index=3 -``` - -在客户端另一终端切换模式并查看指标: - -```bash -printf 'tcp_only\n' > /tmp/brpc-rdma-mode -curl --noproxy '*' 'http://127.0.0.1:8001/vars/rdma_recovery_*' -printf 'auto\n' > /tmp/brpc-rdma-mode -``` - -`short` 每次调用建立连接,`single` 复用已协商的传输。 -CSV 每秒输出累计成功、失败、传输协商、TCP 准入及冷却状态。 -健康阶段检查 `failure=0`;注入阶段记录新增失败和恢复时延。 -`rdma_recovery_status` 提供分原因故障计数,TCP 准入指标按连接累计。 - -## 验收 - -| 场景 | 操作 | 预期 | -| --- | --- | --- | -| 正常通信 | AUTO 运行 | RPC 成功,RDMA 协商数增加 | -| 运行中降级 | 切换 TCP_ONLY | TCP 准入增加,RDMA 协商暂停 | -| 恢复 RDMA | 切回 AUTO | short 连接的 RDMA 协商恢复 | -| 端口故障 | 关闭 RDMA 测试端口 | port_active=false,后续连接使用 TCP | -| 端口恢复 | 恢复端口并等待冷却 | RDMA 准入恢复 | -| 对端故障 | 中断服务端 RDMA 数据路径 | 故障计数增加,对端冷却,后续连接使用 TCP | -| RNR | 暂停接收 WR 补充并突破 credit 保护 | rnr 计数增加,对端冷却 | - -使用 `single` 观察在途失败,使用 `short` 检查新连接的回落和恢复。 - -## URMA - -[URMA 测试入口](../../docs/cn/urma.md#运行期故障恢复)提供恢复客户端及 SEND 失败注入运行器, -支持 v2/v3 握手、模式切换、对端冷却、TCP 准入和 URMA 恢复。 diff --git a/example/urma_performance/CMakeLists.txt b/example/urma_performance/CMakeLists.txt index 7ae1d26316..3b92b83720 100644 --- a/example/urma_performance/CMakeLists.txt +++ b/example/urma_performance/CMakeLists.txt @@ -55,7 +55,7 @@ target_link_libraries(urma_performance_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB} target_link_libraries(urma_performance_server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) add_executable(urma_recovery_client - ../rdma_performance/recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) + recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) brpc_example_configure_target(urma_recovery_client) target_link_libraries(urma_recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/rdma_performance/recovery_client.cpp b/example/urma_performance/recovery_client.cpp similarity index 77% rename from example/rdma_performance/recovery_client.cpp rename to example/urma_performance/recovery_client.cpp index c0673620b7..5c6fc9ed5f 100644 --- a/example/rdma_performance/recovery_client.cpp +++ b/example/urma_performance/recovery_client.cpp @@ -23,17 +23,15 @@ #include "brpc/channel.h" #include "brpc/controller.h" #include "brpc/server.h" -#include "brpc/rdma/rdma_recovery_policy.h" #include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "test.pb.h" -DEFINE_string(server, "127.0.0.1:8002", "Performance test server address"); +DEFINE_string(server, "127.0.0.1:8003", "Performance test server address"); DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); -DEFINE_bool(use_rdma, true, "Use RDMA; false runs a TCP smoke test"); -DEFINE_string(transport, "", "rdma, urma or tcp; empty uses --use_rdma"); +DEFINE_string(transport, "urma", "urma or tcp"); DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); DEFINE_int32(interval_ms, 100, "Pause between RPCs"); DEFINE_int32(timeout_ms, 1000, "RPC deadline"); @@ -42,34 +40,27 @@ DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); DEFINE_int32(attachment_size, 0, "Echo attachment bytes and verify the returned content"); static bool SetMode(const std::string& value) { - using brpc::rdma::RdmaRecoveryMode; - RdmaRecoveryMode mode; + using brpc::urma::UrmaRecoveryMode; + UrmaRecoveryMode mode; if (value == "legacy") { - mode = RdmaRecoveryMode::LEGACY; + mode = UrmaRecoveryMode::LEGACY; } else if (value == "auto") { - mode = RdmaRecoveryMode::AUTO; + mode = UrmaRecoveryMode::AUTO; } else if (value == "tcp_only") { - mode = RdmaRecoveryMode::TCP_ONLY; + mode = UrmaRecoveryMode::TCP_ONLY; } else { return false; } - return FLAGS_transport == "urma" ? brpc::urma::SetUrmaRecoveryMode(mode) : - brpc::rdma::SetRdmaRecoveryMode(mode); + return brpc::urma::SetUrmaRecoveryMode(mode); } int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); - if (FLAGS_transport.empty()) { - FLAGS_transport = FLAGS_use_rdma ? "rdma" : "tcp"; - } - if (FLAGS_transport != "rdma" && FLAGS_transport != "urma" && - FLAGS_transport != "tcp") { + if (FLAGS_transport != "urma" && FLAGS_transport != "tcp") { std::cerr << "Invalid transport\n"; return 1; } - const bool cooldown_ok = FLAGS_transport == "urma" ? - brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms) : - brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms); + const bool cooldown_ok = brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms); if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || FLAGS_attachment_size < 0 || !cooldown_ok || @@ -80,7 +71,7 @@ int main(int argc, char* argv[]) { brpc::Channel channel; brpc::ChannelOptions options; options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA : - FLAGS_transport == "rdma" ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + brpc::SOCKET_MODE_TCP; options.connection_type = FLAGS_connection_type; options.timeout_ms = FLAGS_timeout_ms; options.connect_timeout_ms = FLAGS_timeout_ms; @@ -102,7 +93,7 @@ int main(int argc, char* argv[]) { const int64_t start = butil::monotonic_time_us(); int64_t next_report = start; std::cout << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," - "cooled_peers,port_active,port_cooldown_ms\n"; + "cooled_peers\n"; while (!brpc::IsAskedToQuit() && butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { if (!FLAGS_mode_file.empty()) { @@ -139,14 +130,11 @@ int main(int argc, char* argv[]) { } const int64_t now = butil::monotonic_time_us(); if (now >= next_report) { - const auto s = FLAGS_transport == "urma" ? - brpc::urma::GetUrmaRecoveryStatus() : brpc::rdma::GetRdmaRecoveryStatus(); + const auto s = brpc::urma::GetUrmaRecoveryStatus(); std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded << ',' << failed << ',' << s.negotiated_connections << ',' - << s.tcp_only_admissions + s.port_cooldown_admissions + - s.peer_cooldown_admissions - << ',' << s.cooled_peers << ',' << s.port_active << ',' - << s.port_cooldown_ms << std::endl; + << s.tcp_only_admissions + s.peer_cooldown_admissions + << ',' << s.cooled_peers << std::endl; next_report = now + 1000000; } usleep(static_cast(FLAGS_interval_ms) * 1000); diff --git a/example/urma_performance/run_recovery_injection.py b/example/urma_performance/run_recovery_injection.py index 4e172c80ff..f6725979ad 100644 --- a/example/urma_performance/run_recovery_injection.py +++ b/example/urma_performance/run_recovery_injection.py @@ -45,7 +45,7 @@ def decode_bvar(text): def read_samples(path): with path.open() as stream: return [list(map(int, row[:1] + row[2:])) for row in csv.reader(stream) - if len(row) == 9 and row[0].isdigit()] + if len(row) == 7 and row[0].isdigit()] def wait_for(proc, predicate, timeout, description): diff --git a/example/urma_performance/test_run_recovery_injection.py b/example/urma_performance/test_run_recovery_injection.py index e1d42b3d83..472229b930 100644 --- a/example/urma_performance/test_run_recovery_injection.py +++ b/example/urma_performance/test_run_recovery_injection.py @@ -41,9 +41,9 @@ def test_samples_skip_headers_summary_and_partial_rows(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "client.csv" path.write_text("seconds,mode,success,failure\n" - "0,auto,1,0,1,0,0,1,0\n1,auto," + "0,auto,1,0,1,0,0\n1,auto," "\ncompleted success=1 failure=0\n") - self.assertEqual([[0, 1, 0, 1, 0, 0, 1, 0]], runner.read_samples(path)) + self.assertEqual([[0, 1, 0, 1, 0, 0]], runner.read_samples(path)) def phases(self): baseline = {"mode": "auto", "epoch": 2, "negotiated_connections": 10, diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index cb0fcc9d01..0dc41c9dd1 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -29,7 +29,6 @@ #include "brpc/reloadable_flags.h" #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_helper.h" -#include "brpc/rdma/rdma_recovery_policy.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma_transport.h" #include "brpc/rdma/rdma_handshake.h" @@ -132,8 +131,6 @@ RdmaEndpoint::RdmaEndpoint(Socket* s) : _socket(s) , _state(UNINIT) , _handshake_version(0) - , _recovery_epoch(0) - , _runtime_failure(RdmaFailureReason::NONE) , _resource(nullptr) , _send_cq_events(0) , _recv_cq_events(0) @@ -181,8 +178,6 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; - _recovery_epoch = 0; - _runtime_failure.store(RdmaFailureReason::NONE, butil::memory_order_relaxed); _outgoing_ece.reset(); _resource = nullptr; _send_cq_events = 0; @@ -217,10 +212,7 @@ void RdmaConnect::StartConnect(const Socket* socket, if (Socket::Address(socket->id(), &s) != 0) { return; } - if (!IsRdmaAvailable() || - !GlobalRdmaRecoveryPolicy().AllowNewConnection( - butil::endpoint2str(socket->remote_side()).c_str(), - &rdma_transport->_rdma_ep->_recovery_epoch)) { + if (!IsRdmaAvailable()) { rdma_transport->_rdma_state = RdmaTransport::RDMA_OFF; rdma_transport->_rdma_ep->_state.store( RdmaEndpoint::FALLBACK_TCP, butil::memory_order_release); @@ -244,50 +236,6 @@ void RdmaConnect::StartConnect(const Socket* socket, void RdmaConnect::StopConnect(Socket* socket) { } -void RdmaEndpoint::ReportFailure(RdmaFailureReason reason) { - RdmaFailureReason expected = RdmaFailureReason::NONE; - if (reason == RdmaFailureReason::NONE || - !_runtime_failure.compare_exchange_strong(expected, reason)) { - return; - } - if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { - return; - } - GlobalRdmaRecoveryPolicy().ReportPeerFailure( - _socket->CreatedByConnect() ? - butil::endpoint2str(_socket->remote_side()).c_str() : "", - _recovery_epoch, reason); - LOG(WARNING) << "RDMA runtime failure=" << RdmaFailureReasonName(reason) - << " socket=" << _socket->id() - << " port=" << static_cast(GetRdmaPortNum()) - << " epoch=" << _recovery_epoch; -} - -void RdmaEndpoint::OnAsyncError(Socket* socket, const ibv_async_event& event) { - auto* transport = dynamic_cast(socket->_transport.get()); - RdmaEndpoint* ep = transport ? transport->GetRdmaEp() : nullptr; - if (!ep || !ep->_resource) { - return; - } - RdmaFailureReason reason; - if (event.event_type == IBV_EVENT_CQ_ERR) { - if (event.element.cq != ep->_resource->send_cq && - event.element.cq != ep->_resource->recv_cq && - event.element.cq != ep->_resource->polling_cq) { - return; - } - reason = RdmaFailureReason::CQ_ERROR; - } else { - if (event.element.qp != ep->_resource->qp) { - return; - } - reason = event.event_type == IBV_EVENT_QP_FATAL ? - RdmaFailureReason::QP_FATAL : RdmaFailureReason::ACCESS_ERROR; - } - ep->ReportFailure(reason); - socket->SetFailed(ERDMA, "RDMA async error: %s", RdmaFailureReasonName(reason)); -} - void RdmaConnect::Run() { _done(errno, _data); } @@ -633,7 +581,6 @@ void* RdmaEndpoint::ProcessHandshakeAtClient(void* arg) { if (rdma_transport->_rdma_state == RdmaTransport::RDMA_ON) { ep->_state.store(ESTABLISHED, butil::memory_order_release); - GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); // The handshake is over, so the QP stream may start parsing now. if (ep->StartCqEvents() < 0) { const int saved_errno = errno; @@ -736,9 +683,7 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s } // Negotiate + allocate resources. - bool negotiated = r == RemoteHelloResult::NEGOTIATED && - GlobalRdmaRecoveryPolicy().AllowNewConnection( - "", &ep->_recovery_epoch); + bool negotiated = r == RemoteHelloResult::NEGOTIATED; if (negotiated) { ep->ApplyRemoteHello(remote); ep->_state.store(S_ALLOC_QPCQ, butil::memory_order_relaxed); @@ -820,7 +765,6 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s << ") on " << s->description(); rdma_transport->_rdma_state = RdmaTransport::RDMA_ON; ep->_state.store(ESTABLISHED, butil::memory_order_release); - GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); s->reset_parsing_context(nullptr); // Two things are deliberately not done here. @@ -1033,9 +977,6 @@ ssize_t RdmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { std::ostringstream oss; DebugInfo(oss, ", "); LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); - if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { - ReportFailure(RdmaFailureReason::SEND_POST); - } errno = err; return -1; } @@ -1086,10 +1027,6 @@ int RdmaEndpoint::SendImm(uint32_t imm) { // We use other way to guarantee the Send Queue is not full. // So we just consider this error as an unrecoverable error. LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); - if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { - ReportFailure(RdmaFailureReason::SEND_POST); - } - errno = err; return -1; } @@ -1107,7 +1044,8 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (0 == wc.wr_id) { _sq_imm_window_size += 1; // If there are any unacknowledged recvs, send an ack. - return SendAck(0); + SendAck(0); + return 0; } // Update SQ window. uint16_t wnd_to_update = wc.wr_id; @@ -1160,8 +1098,8 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (wc.byte_len > 0 && SendAck(1) < 0) { - return -1; + if (wc.byte_len > 0) { + SendAck(1); } return wc.byte_len; } @@ -1189,8 +1127,6 @@ int RdmaEndpoint::DoPostRecv(void* block, size_t block_size) { int err = ibv_post_recv(_resource->qp, &wr, &bad); if (err != 0) { LOG(WARNING) << "Fail to ibv_post_recv: " << berror(err); - ReportFailure(RdmaFailureReason::RECV_POST); - errno = err; return -1; } return 0; @@ -1398,8 +1334,6 @@ int RdmaEndpoint::DoAllocateResources() { return -1; } - BindResourceToSocket(); - if (!FLAGS_rdma_use_polling) { if (0 != ReqNotifyCq(true, false)) { return -1; @@ -1425,20 +1359,6 @@ int RdmaEndpoint::DoAllocateResources() { return 0; } -void RdmaEndpoint::BindResourceToSocket() { - void* owner = reinterpret_cast(_socket->id()); - _resource->qp->qp_context = owner; - if (_resource->send_cq) { - _resource->send_cq->cq_context = owner; - } - if (_resource->recv_cq) { - _resource->recv_cq->cq_context = owner; - } - if (_resource->polling_cq) { - _resource->polling_cq->cq_context = owner; - } -} - int RdmaEndpoint::StartCqEvents() { if (InputMessengerProcessor::STREAM_NONE != _socket->parsing_stream_type()) { LOG(WARNING) << "StartCqEvents() called while " << *_socket << " is parsing"; @@ -1624,8 +1544,7 @@ void RdmaEndpoint::DeallocateResources() { PollerRemoveCqSid(); } bool move_to_rdma_resource_list = false; - if (_runtime_failure.load(butil::memory_order_relaxed) == RdmaFailureReason::NONE && - _sq_size <= FLAGS_rdma_prepared_qp_size && + if (_sq_size <= FLAGS_rdma_prepared_qp_size && _rq_size <= FLAGS_rdma_prepared_qp_size && FLAGS_rdma_prepared_qp_cnt > 0) { ibv_qp_attr attr; @@ -1732,7 +1651,6 @@ int RdmaEndpoint::GetAndAckEvents(SocketUniquePtr& s) { if (errno != EAGAIN) { const int saved_errno = errno; PLOG(ERROR) << "Fail to get cq event from " << s->description(); - ReportFailure(RdmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to get cq event from %s: %s", s->description().c_str(), berror(saved_errno)); return -1; @@ -1773,7 +1691,6 @@ int RdmaEndpoint::ReqNotifyCq(bool send_cq, bool fatal_on_error) { PLOG(WARNING) << "Fail to arm " << (send_cq ? "send" : "recv") << " CQ comp channel from " << _socket->description(); if (fatal_on_error) { - ReportFailure(RdmaFailureReason::CQ_NOTIFY); _socket->SetFailed(err, "Fail to arm %s CQ channel from %s: %s", send_cq ? "send" : "recv", _socket->description().c_str(), berror(err)); @@ -1827,7 +1744,6 @@ void RdmaEndpoint::PollCq(Socket* m) { if (cnt < 0) { const int saved_errno = errno; PLOG(WARNING) << "Fail to poll cq: " << s->description(); - ep->ReportFailure(RdmaFailureReason::CQ_POLL); s->SetFailed(saved_errno, "Fail to poll cq from %s: %s", s->description().c_str(), berror(saved_errno)); return; @@ -1896,7 +1812,6 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { - ep->ReportFailure(ClassifyRdmaCompletion(wc[i].status)); PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); s->SetFailed(ERDMA, "RDMA completion error(%d) from %s: %s", diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index 85d3b280c9..eda5e558ef 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -32,7 +32,6 @@ #include "butil/containers/optional.h" #include "brpc/socket.h" #include "brpc/rdma/rdma_handshake_server.h" -#include "brpc/rdma/rdma_recovery_policy.h" namespace brpc { @@ -137,9 +136,6 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // Real handshake for RDMA-mode sockets. static ParseResult ExecuteServerHandshake(butil::IOBuf* source, Socket* socket); - // Called with an addressed Socket held until its async event is acknowledged. - static void OnAsyncError(Socket* socket, const ibv_async_event& event); - // Initialize polling mode static int PollingModeInitialize(bthread_tag_t tag, std::function callback, @@ -149,9 +145,6 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); static void PollingModeRelease(bthread_tag_t tag); private: - void ReportFailure(RdmaFailureReason reason); - void BindResourceToSocket(); - enum State { UNINIT = 0x0, C_ALLOC_QPCQ = 0x1, @@ -314,10 +307,6 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // 3 = v3 "RDM3" int _handshake_version; - // Admission epoch used to discard failures from an earlier recovery mode. - uint64_t _recovery_epoch; - butil::atomic _runtime_failure; - // ECE payload to advertise in the next local hello: // Client: the locally queried ECE capabilities (filled // before C_HELLO_SEND); diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index dc0e622a75..7e671a85ba 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -30,7 +30,6 @@ #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma/rdma_helper.h" -#include "brpc/rdma/rdma_recovery_policy.h" namespace butil { @@ -269,7 +268,7 @@ static void OnRdmaAsyncEvent(Socket* m) { SocketId sid = (SocketId)event.element.qp->qp_context; SocketUniquePtr s; if (Socket::Address(sid, &s) == 0) { - RdmaEndpoint::OnAsyncError(s.get(), event); + s->SetFailed(ERDMA, "QP fatal error"); LOG(WARNING) << "Receive a QP fatal error on " << s->description(); } // NOTE: @@ -281,11 +280,7 @@ static void OnRdmaAsyncEvent(Socket* m) { break; } case IBV_EVENT_CQ_ERR: { - SocketUniquePtr s; - const SocketId sid = reinterpret_cast(event.element.cq->cq_context); - if (Socket::Address(sid, &s) == 0) { - RdmaEndpoint::OnAsyncError(s.get(), event); - } + LOG(WARNING) << "CQ overruns, the connection will be stopped."; IbvAckAsyncEvent(&event); break; } @@ -318,10 +313,8 @@ static void OnRdmaAsyncEvent(Socket* m) { } case IBV_EVENT_PORT_ACTIVE: case IBV_EVENT_PORT_ERR: { - if (event.element.port_num == g_port_num) { - GlobalRdmaRecoveryPolicy().ReportPortState( - event.event_type == IBV_EVENT_PORT_ACTIVE); - } + // Port up/down will lead these two events. + // The port error is recoverable. IbvAckAsyncEvent(&event); break; } @@ -706,31 +699,6 @@ bool IsRdmaAvailable() { return g_rdma_available.load(butil::memory_order_acquire); } -bool IsRdmaPathFailure(ibv_wc_status status) { - const auto reason = ClassifyRdmaCompletion(status); - return reason != RdmaFailureReason::NONE && - reason != RdmaFailureReason::ACCESS_ERROR; -} - -RdmaFailureReason ClassifyRdmaCompletion(ibv_wc_status status) { - switch (status) { - case IBV_WC_SUCCESS: - case IBV_WC_WR_FLUSH_ERR: - return RdmaFailureReason::NONE; - case IBV_WC_RETRY_EXC_ERR: - return RdmaFailureReason::RETRY_EXHAUSTED; - case IBV_WC_RNR_RETRY_EXC_ERR: - return RdmaFailureReason::RNR; - case IBV_WC_LOC_PROT_ERR: - case IBV_WC_LOC_ACCESS_ERR: - case IBV_WC_REM_ACCESS_ERR: - case IBV_WC_MW_BIND_ERR: - return RdmaFailureReason::ACCESS_ERROR; - default: - return RdmaFailureReason::TRANSPORT_ERROR; - } -} - void GlobalDisableRdma() { if (g_rdma_available.exchange(false, butil::memory_order_acquire)) { LOG(FATAL) << "RDMA is disabled due to some unrecoverable problem"; diff --git a/src/brpc/rdma/rdma_helper.h b/src/brpc/rdma/rdma_helper.h index 5f49f5779f..2383d2d8b4 100644 --- a/src/brpc/rdma/rdma_helper.h +++ b/src/brpc/rdma/rdma_helper.h @@ -29,8 +29,6 @@ namespace brpc { namespace rdma { -enum class RdmaFailureReason; - // Initialize RDMA environment // Exit if failed void GlobalRdmaInitializeOrDie(); @@ -91,10 +89,6 @@ bool IsRdmaAvailable(); // Disable RDMA in the remaining lifetime of the process void GlobalDisableRdma(); -// Completion failures that indicate an unusable path or exhausted receive credits. -bool IsRdmaPathFailure(ibv_wc_status status); -RdmaFailureReason ClassifyRdmaCompletion(ibv_wc_status status); - // If the given protocol supported by RDMA bool SupportedByRdma(std::string protocol); diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h deleted file mode 100644 index c46c8a0705..0000000000 --- a/src/brpc/rdma/rdma_recovery_policy.h +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#ifndef BRPC_RDMA_RECOVERY_POLICY_H -#define BRPC_RDMA_RECOVERY_POLICY_H - -#include -#include -#include -#include -#include - -namespace brpc { -namespace rdma { - -enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; - -enum class RdmaFailureReason { - NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, - CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, TRANSPORT_ERROR, RECV_POST, COUNT -}; - -const char* RdmaFailureReasonName(RdmaFailureReason reason); - -struct RdmaRecoveryStatus { - RdmaRecoveryMode mode = RdmaRecoveryMode::LEGACY; - uint64_t epoch = 0; - bool port_active = true; - int64_t port_cooldown_ms = 0; - size_t cooled_peers = 0; - uint64_t failures[static_cast(RdmaFailureReason::COUNT)] = {}; - uint64_t tcp_only_admissions = 0; - uint64_t port_cooldown_admissions = 0; - uint64_t peer_cooldown_admissions = 0; - uint64_t port_errors = 0; - uint64_t cache_evictions = 0; - uint64_t negotiated_connections = 0; -}; - -// Admission policy for new connections on the process's selected RDMA port. -// An empty peer selects server admission, which only checks the local port. -class RdmaRecoveryPolicy { -public: - typedef int64_t (*Clock)(); - explicit RdmaRecoveryPolicy(Clock clock, size_t max_peers = 1024); - bool SetMode(RdmaRecoveryMode mode); - RdmaRecoveryMode GetMode(); - // Positive milliseconds, at most INT32_MAX. Applies to future failures. - bool SetCooldownMs(int64_t cooldown_ms); - bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); - void ReportPeerFailure(const std::string& peer, uint64_t epoch, - RdmaFailureReason reason = RdmaFailureReason::RETRY_EXHAUSTED); - void ReportPortState(bool active); - RdmaRecoveryStatus GetStatus(); - void RecordNegotiatedConnection(uint64_t epoch); - -private: - std::mutex _mutex; - const Clock _clock; - const size_t _max_peers; - RdmaRecoveryMode _mode; - uint64_t _epoch; - int64_t _cooldown_us; - bool _port_active; - int64_t _port_retry_at; - std::map _peers; - RdmaRecoveryStatus _status; -}; - -// Process-lifetime policy, shared by client and server handshake admission. -RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy(); -bool SetRdmaRecoveryMode(RdmaRecoveryMode mode); -RdmaRecoveryMode GetRdmaRecoveryMode(); -bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms); -RdmaRecoveryStatus GetRdmaRecoveryStatus(); - -} // namespace rdma -} // namespace brpc - -#endif // BRPC_RDMA_RECOVERY_POLICY_H diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index a2f163a349..d3290dcb3a 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -1690,7 +1690,7 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) << " recovery_epoch=" << _recovery_epoch - << " runtime_failure=" << rdma::RdmaFailureReasonName(_runtime_failure.load()) + << " runtime_failure=" << UrmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/urma/urma_recovery_policy.cpp similarity index 54% rename from src/brpc/rdma/rdma_recovery_policy.cpp rename to src/brpc/urma/urma_recovery_policy.cpp index c8329a95a1..157ae02a79 100644 --- a/src/brpc/rdma/rdma_recovery_policy.cpp +++ b/src/brpc/urma/urma_recovery_policy.cpp @@ -15,45 +15,42 @@ // specific language governing permissions and limitations // under the License. -#include "brpc/rdma/rdma_recovery_policy.h" #include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "bvar/passive_status.h" namespace brpc { -namespace rdma { +namespace urma { -const char* RdmaFailureReasonName(RdmaFailureReason reason) { +const char* UrmaFailureReasonName(UrmaFailureReason reason) { switch (reason) { - case RdmaFailureReason::NONE: return "none"; - case RdmaFailureReason::RETRY_EXHAUSTED: return "retry_exhausted"; - case RdmaFailureReason::RNR: return "rnr"; - case RdmaFailureReason::QP_FATAL: return "qp_fatal"; - case RdmaFailureReason::CQ_ERROR: return "cq_error"; - case RdmaFailureReason::CQ_POLL: return "cq_poll"; - case RdmaFailureReason::CQ_EVENT: return "cq_event"; - case RdmaFailureReason::CQ_NOTIFY: return "cq_notify"; - case RdmaFailureReason::SEND_POST: return "send_post"; - case RdmaFailureReason::ACCESS_ERROR: return "access_error"; - case RdmaFailureReason::TRANSPORT_ERROR: return "transport_error"; - case RdmaFailureReason::RECV_POST: return "recv_post"; + case UrmaFailureReason::NONE: return "none"; + case UrmaFailureReason::RETRY_EXHAUSTED: return "retry_exhausted"; + case UrmaFailureReason::RNR: return "rnr"; + case UrmaFailureReason::QP_FATAL: return "qp_fatal"; + case UrmaFailureReason::CQ_ERROR: return "cq_error"; + case UrmaFailureReason::CQ_POLL: return "cq_poll"; + case UrmaFailureReason::CQ_EVENT: return "cq_event"; + case UrmaFailureReason::CQ_NOTIFY: return "cq_notify"; + case UrmaFailureReason::SEND_POST: return "send_post"; + case UrmaFailureReason::ACCESS_ERROR: return "access_error"; + case UrmaFailureReason::TRANSPORT_ERROR: return "transport_error"; + case UrmaFailureReason::RECV_POST: return "recv_post"; default: return "unknown"; } } -RdmaRecoveryPolicy::RdmaRecoveryPolicy(Clock clock, size_t max_peers) +UrmaRecoveryPolicy::UrmaRecoveryPolicy(Clock clock, size_t max_peers) : _clock(clock) , _max_peers(max_peers) - , _mode(RdmaRecoveryMode::LEGACY) + , _mode(UrmaRecoveryMode::LEGACY) , _epoch(1) - , _cooldown_us(5000000) - , _port_active(true) - , _port_retry_at(0) { + , _cooldown_us(5000000) { } -bool RdmaRecoveryPolicy::SetMode(RdmaRecoveryMode mode) { - if (mode != RdmaRecoveryMode::LEGACY && mode != RdmaRecoveryMode::AUTO && - mode != RdmaRecoveryMode::TCP_ONLY) { +bool UrmaRecoveryPolicy::SetMode(UrmaRecoveryMode mode) { + if (mode != UrmaRecoveryMode::LEGACY && mode != UrmaRecoveryMode::AUTO && + mode != UrmaRecoveryMode::TCP_ONLY) { return false; } std::lock_guard lock(_mutex); @@ -65,12 +62,12 @@ bool RdmaRecoveryPolicy::SetMode(RdmaRecoveryMode mode) { return true; } -RdmaRecoveryMode RdmaRecoveryPolicy::GetMode() { +UrmaRecoveryMode UrmaRecoveryPolicy::GetMode() { std::lock_guard lock(_mutex); return _mode; } -bool RdmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { +bool UrmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { if (cooldown_ms <= 0 || cooldown_ms > INT32_MAX) { return false; } @@ -79,24 +76,20 @@ bool RdmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { return true; } -bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, - uint64_t* epoch) { +bool UrmaRecoveryPolicy::AllowNewConnection(const std::string& peer, + uint64_t* epoch) { std::lock_guard lock(_mutex); if (epoch != nullptr) { *epoch = _epoch; } - if (_mode == RdmaRecoveryMode::LEGACY) { + if (_mode == UrmaRecoveryMode::LEGACY) { return true; } - if (_mode == RdmaRecoveryMode::TCP_ONLY) { + if (_mode == UrmaRecoveryMode::TCP_ONLY) { ++_status.tcp_only_admissions; return false; } const int64_t now = _clock(); - if (!_port_active || now < _port_retry_at) { - ++_status.port_cooldown_admissions; - return false; - } auto it = _peers.find(peer); if (it == _peers.end()) { return true; @@ -109,17 +102,17 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, return true; } -void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, +void UrmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, uint64_t epoch, - RdmaFailureReason reason) { + UrmaFailureReason reason) { std::lock_guard lock(_mutex); - if (epoch != _epoch || reason <= RdmaFailureReason::NONE || - reason >= RdmaFailureReason::COUNT) { + if (epoch != _epoch || reason <= UrmaFailureReason::NONE || + reason >= UrmaFailureReason::COUNT) { return; } ++_status.failures[static_cast(reason)]; - if (_mode != RdmaRecoveryMode::AUTO || peer.empty() || _max_peers == 0 || - reason == RdmaFailureReason::ACCESS_ERROR) { + if (_mode != UrmaRecoveryMode::AUTO || peer.empty() || _max_peers == 0 || + reason == UrmaFailureReason::ACCESS_ERROR) { return; } const int64_t now = _clock(); @@ -143,31 +136,19 @@ void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, _peers[peer] = now + _cooldown_us; } -void RdmaRecoveryPolicy::ReportPortState(bool active) { - std::lock_guard lock(_mutex); - _port_active = active; - if (!active) { - ++_status.port_errors; - _port_retry_at = _clock() + _cooldown_us; - } -} - -RdmaRecoveryStatus RdmaRecoveryPolicy::GetStatus() { +UrmaRecoveryStatus UrmaRecoveryPolicy::GetStatus() { std::lock_guard lock(_mutex); - RdmaRecoveryStatus result = _status; + UrmaRecoveryStatus result = _status; result.mode = _mode; result.epoch = _epoch; - result.port_active = _port_active; const int64_t now = _clock(); - result.port_cooldown_ms = _port_retry_at > now ? - (_port_retry_at - now + 999) / 1000 : 0; for (const auto& peer : _peers) { result.cooled_peers += peer.second > now; } return result; } -void RdmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { +void UrmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { std::lock_guard lock(_mutex); if (epoch == _epoch) { ++_status.negotiated_connections; @@ -175,79 +156,44 @@ void RdmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { } static void DescribeRecovery(std::ostream& os, void* arg) { - const auto s = static_cast(arg)->GetStatus(); + const auto s = static_cast(arg)->GetStatus(); os << "{\"mode\":\"" - << (s.mode == RdmaRecoveryMode::AUTO ? "auto" : - s.mode == RdmaRecoveryMode::TCP_ONLY ? "tcp_only" : "legacy") + << (s.mode == UrmaRecoveryMode::AUTO ? "auto" : + s.mode == UrmaRecoveryMode::TCP_ONLY ? "tcp_only" : "legacy") << "\",\"epoch\":" << s.epoch - << ",\"port_active\":" << (s.port_active ? "true" : "false") - << ",\"port_cooldown_ms\":" << s.port_cooldown_ms << ",\"cooled_peers\":" << s.cooled_peers << ",\"tcp_only_admissions\":" << s.tcp_only_admissions - << ",\"port_cooldown_admissions\":" << s.port_cooldown_admissions << ",\"peer_cooldown_admissions\":" << s.peer_cooldown_admissions - << ",\"port_errors\":" << s.port_errors << ",\"cache_evictions\":" << s.cache_evictions << ",\"negotiated_connections\":" << s.negotiated_connections << ",\"failures\":{"; - for (size_t i = 1; i < static_cast(RdmaFailureReason::COUNT); ++i) { + for (size_t i = 1; i < static_cast(UrmaFailureReason::COUNT); ++i) { if (i > 1) { os << ','; } - os << '"' << RdmaFailureReasonName(static_cast(i)) + os << '"' << UrmaFailureReasonName(static_cast(i)) << "\":" << s.failures[i]; } os << "}}"; } struct RecoveryMetrics { - RecoveryMetrics(RdmaRecoveryPolicy* policy, const char* status_name, + RecoveryMetrics(UrmaRecoveryPolicy* policy, const char* status_name, const char* fallback_name) : status(status_name, DescribeRecovery, policy) , fallbacks(fallback_name, [](void* arg) { - const auto s = static_cast(arg)->GetStatus(); - return s.tcp_only_admissions + s.port_cooldown_admissions + - s.peer_cooldown_admissions; + const auto s = static_cast(arg)->GetStatus(); + return s.tcp_only_admissions + s.peer_cooldown_admissions; }, policy) { } bvar::PassiveStatus status; bvar::PassiveStatus fallbacks; }; -RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { - // Async event callbacks may outlive ordinary static destructors. - static RdmaRecoveryPolicy* policy = - new RdmaRecoveryPolicy(butil::monotonic_time_us); - static RecoveryMetrics* metrics = new RecoveryMetrics( - policy, "rdma_recovery_status", "rdma_recovery_tcp_fallback_admissions"); - (void)metrics; - return *policy; -} - -bool SetRdmaRecoveryMode(RdmaRecoveryMode mode) { - return GlobalRdmaRecoveryPolicy().SetMode(mode); -} - -RdmaRecoveryMode GetRdmaRecoveryMode() { - return GlobalRdmaRecoveryPolicy().GetMode(); -} - -bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms) { - return GlobalRdmaRecoveryPolicy().SetCooldownMs(cooldown_ms); -} - -RdmaRecoveryStatus GetRdmaRecoveryStatus() { - return GlobalRdmaRecoveryPolicy().GetStatus(); -} - -} // namespace rdma - -namespace urma { - UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy() { static UrmaRecoveryPolicy* policy = new UrmaRecoveryPolicy(butil::monotonic_time_us); - static rdma::RecoveryMetrics* metrics = new rdma::RecoveryMetrics( + static RecoveryMetrics* metrics = new RecoveryMetrics( policy, "urma_recovery_status", "urma_recovery_tcp_fallback_admissions"); (void)metrics; return *policy; diff --git a/src/brpc/urma/urma_recovery_policy.h b/src/brpc/urma/urma_recovery_policy.h index da2792fd54..e9cbbf5005 100644 --- a/src/brpc/urma/urma_recovery_policy.h +++ b/src/brpc/urma/urma_recovery_policy.h @@ -18,17 +18,63 @@ #ifndef BRPC_URMA_RECOVERY_POLICY_H #define BRPC_URMA_RECOVERY_POLICY_H -#include "brpc/rdma/rdma_recovery_policy.h" +#include +#include +#include +#include +#include namespace brpc { namespace urma { -// Share the admission algorithm while keeping transport state independent. -using UrmaRecoveryMode = rdma::RdmaRecoveryMode; -using UrmaFailureReason = rdma::RdmaFailureReason; -using UrmaRecoveryStatus = rdma::RdmaRecoveryStatus; -using UrmaRecoveryPolicy = rdma::RdmaRecoveryPolicy; +enum class UrmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; +enum class UrmaFailureReason { + NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, + CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, TRANSPORT_ERROR, RECV_POST, COUNT +}; + +const char* UrmaFailureReasonName(UrmaFailureReason reason); + +struct UrmaRecoveryStatus { + UrmaRecoveryMode mode = UrmaRecoveryMode::LEGACY; + uint64_t epoch = 0; + size_t cooled_peers = 0; + uint64_t failures[static_cast(UrmaFailureReason::COUNT)] = {}; + uint64_t tcp_only_admissions = 0; + uint64_t peer_cooldown_admissions = 0; + uint64_t cache_evictions = 0; + uint64_t negotiated_connections = 0; +}; + +// Admission policy for new URMA connections. +// An empty peer selects server admission, which only checks the mode. +class UrmaRecoveryPolicy { +public: + typedef int64_t (*Clock)(); + explicit UrmaRecoveryPolicy(Clock clock, size_t max_peers = 1024); + bool SetMode(UrmaRecoveryMode mode); + UrmaRecoveryMode GetMode(); + // Positive milliseconds, at most INT32_MAX. Applies to future failures. + bool SetCooldownMs(int64_t cooldown_ms); + bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); + void ReportPeerFailure(const std::string& peer, uint64_t epoch, + UrmaFailureReason reason = UrmaFailureReason::RETRY_EXHAUSTED); + UrmaRecoveryStatus GetStatus(); + void RecordNegotiatedConnection(uint64_t epoch); + +private: + std::mutex _mutex; + const Clock _clock; + const size_t _max_peers; + UrmaRecoveryMode _mode; + uint64_t _epoch; + int64_t _cooldown_us; + std::map _peers; + UrmaRecoveryStatus _status; +}; + +// Process-lifetime policy, shared by client and server handshake admission. UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy(); bool SetUrmaRecoveryMode(UrmaRecoveryMode mode); UrmaRecoveryMode GetUrmaRecoveryMode(); diff --git a/test/brpc_rdma_error_recovery_unittest.cpp b/test/brpc_rdma_error_recovery_unittest.cpp deleted file mode 100644 index 1eaaaf5d71..0000000000 --- a/test/brpc_rdma_error_recovery_unittest.cpp +++ /dev/null @@ -1,220 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include -#include -#include -#include - -#if BRPC_WITH_RDMA -#include "brpc/global.h" -#include "brpc/input_messenger.h" -#include "brpc/rdma/rdma_endpoint.h" -#include "brpc/rdma/rdma_helper.h" -#include "brpc/rdma/rdma_recovery_policy.h" -#include "brpc/rdma_transport.h" -#include "brpc/socket.h" - -namespace brpc { -namespace rdma { -extern bool g_skip_rdma_init; -extern butil::atomic g_rdma_available; -} // namespace rdma -} // namespace brpc - -namespace { -using namespace brpc; - -struct CompletionSource { - ibv_wc_status status; - bool delivered = false; -}; - -int PollError(ibv_cq* cq, int, ibv_wc* wc) { - auto* source = static_cast(cq->cq_context); - if (source->delivered) { - return 0; - } - source->delivered = true; - std::memset(wc, 0, sizeof(*wc)); - wc->status = source->status; - return 1; -} - -int FailRecvPost(ibv_qp*, ibv_recv_wr*, ibv_recv_wr**) { - return EIO; -} - -class RdmaErrorRecoveryTest : public testing::Test { -protected: - void SetUp() override { - GlobalInitializeOrDie(); - _saved_mode = rdma::GetRdmaRecoveryMode(); - ResetPolicy(); - } - - void TearDown() override { - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(_saved_mode); - } - - void ResetPolicy() { - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); - } - - void CreateSocket(SocketUniquePtr* socket) { - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - ASSERT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &options.remote_side)); - options.user = get_client_side_messenger(); - SocketId id; - ASSERT_EQ(0, Socket::Create(options, &id)); - ASSERT_EQ(0, Socket::Address(id, socket)); - } - - static rdma::RdmaEndpoint* Endpoint(const SocketUniquePtr& socket) { - return static_cast(socket->_transport.get())->GetRdmaEp(); - } - - void CheckCompletion(ibv_wc_status status, rdma::RdmaFailureReason reason, - bool cooldown) { - ResetPolicy(); - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - auto* endpoint = Endpoint(socket); - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - ASSERT_TRUE(policy.AllowNewConnection(_peer, &endpoint->_recovery_epoch)); - const auto before = policy.GetStatus(); - SocketOptions carrier_options; - carrier_options.user = endpoint; - SocketId carrier_id; - ASSERT_EQ(0, Socket::Create(carrier_options, &carrier_id)); - SocketUniquePtr carrier; - ASSERT_EQ(0, Socket::Address(carrier_id, &carrier)); - - CompletionSource source{status}; - ibv_context context{}; - context.ops.poll_cq = PollError; - ibv_cq cq{}; - cq.context = &context; - cq.cq_context = &source; - rdma::RdmaResource resource; - resource.polling_cq = &cq; - endpoint->_resource = &resource; - endpoint->_cq_sid = carrier_id; - endpoint->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - rdma::FLAGS_rdma_use_polling = true; - - rdma::RdmaEndpoint::PollCq(carrier.get()); - EXPECT_TRUE(source.delivered); - EXPECT_TRUE(socket->Failed()); - EXPECT_EQ(reason, endpoint->_runtime_failure.load()); - EXPECT_EQ(!cooldown, policy.AllowNewConnection(_peer)); - EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); - const auto after = policy.GetStatus(); - for (size_t i = 1; i < static_cast(rdma::RdmaFailureReason::COUNT); ++i) { - EXPECT_EQ(before.failures[i] + (i == static_cast(reason)), - after.failures[i]); - } - if (reason != rdma::RdmaFailureReason::NONE) { - endpoint->ReportFailure(rdma::RdmaFailureReason::CQ_ERROR); - EXPECT_EQ(after.failures[static_cast(rdma::RdmaFailureReason::CQ_ERROR)], - policy.GetStatus().failures[ - static_cast(rdma::RdmaFailureReason::CQ_ERROR)]); - } - - endpoint->_resource = nullptr; - endpoint->_cq_sid = INVALID_SOCKET_ID; - resource.polling_cq = nullptr; - carrier->_user = nullptr; - carrier->SetFailed(); - } - - const std::string _peer = "127.0.0.1:8123"; - GFLAGS_NAMESPACE::FlagSaver _flags; - rdma::RdmaRecoveryMode _saved_mode; -}; - -TEST_F(RdmaErrorRecoveryTest, fatal_timeout_and_general_completions_cool_peer) { - for (auto status : {IBV_WC_FATAL_ERR, IBV_WC_RESP_TIMEOUT_ERR, IBV_WC_GENERAL_ERR}) { - ASSERT_NO_FATAL_FAILURE(CheckCompletion( - status, rdma::RdmaFailureReason::TRANSPORT_ERROR, true)); - } -} - -TEST_F(RdmaErrorRecoveryTest, retry_and_rnr_completions_cool_peer) { - ASSERT_NO_FATAL_FAILURE(CheckCompletion( - IBV_WC_RETRY_EXC_ERR, rdma::RdmaFailureReason::RETRY_EXHAUSTED, true)); - ASSERT_NO_FATAL_FAILURE(CheckCompletion( - IBV_WC_RNR_RETRY_EXC_ERR, rdma::RdmaFailureReason::RNR, true)); -} - -TEST_F(RdmaErrorRecoveryTest, access_and_flush_completions_preserve_admission) { - for (auto status : {IBV_WC_LOC_PROT_ERR, IBV_WC_LOC_ACCESS_ERR, - IBV_WC_REM_ACCESS_ERR, IBV_WC_MW_BIND_ERR}) { - ASSERT_NO_FATAL_FAILURE(CheckCompletion( - status, rdma::RdmaFailureReason::ACCESS_ERROR, false)); - } - ASSERT_NO_FATAL_FAILURE(CheckCompletion( - IBV_WC_WR_FLUSH_ERR, rdma::RdmaFailureReason::NONE, false)); -} - -TEST_F(RdmaErrorRecoveryTest, receive_post_error_preserves_errno_and_reports_once) { - for (bool established : {false, true}) { - ResetPolicy(); - SocketUniquePtr socket; - ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); - auto* endpoint = Endpoint(socket); - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - ASSERT_TRUE(policy.AllowNewConnection(_peer, &endpoint->_recovery_epoch)); - const auto before = policy.GetStatus(); - ibv_context context{}; - context.ops.post_recv = FailRecvPost; - ibv_qp qp{}; - qp.context = &context; - rdma::RdmaResource resource; - resource.qp = &qp; - endpoint->_resource = &resource; - endpoint->_state.store(established ? rdma::RdmaEndpoint::ESTABLISHED : - rdma::RdmaEndpoint::C_ALLOC_QPCQ); - char data[32]{}; - for (int attempt = 0; attempt < 2; ++attempt) { - errno = EALREADY; - EXPECT_EQ(-1, endpoint->DoPostRecv(data, sizeof(data))); - EXPECT_EQ(EIO, errno); - } - EXPECT_EQ(!established, policy.AllowNewConnection(_peer)); - const size_t index = static_cast(rdma::RdmaFailureReason::RECV_POST); - EXPECT_EQ(before.failures[index] + established, policy.GetStatus().failures[index]); - endpoint->_resource = nullptr; - resource.qp = nullptr; - socket->SetFailed(); - } -} -} // namespace -#endif // BRPC_WITH_RDMA - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); -#if BRPC_WITH_RDMA - brpc::rdma::g_rdma_available.store(true, butil::memory_order_relaxed); - brpc::rdma::g_skip_rdma_init = true; -#endif - return RUN_ALL_TESTS(); -} diff --git a/test/brpc_rdma_runtime_unittest.cpp b/test/brpc_rdma_runtime_unittest.cpp deleted file mode 100644 index a1db214a5a..0000000000 --- a/test/brpc_rdma_runtime_unittest.cpp +++ /dev/null @@ -1,510 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - - -#include -#include -#include -#include -#if BRPC_WITH_RDMA -#include -#include -#include -#include -#include -#include "butil/endpoint.h" -#include "butil/fd_guard.h" -#include "butil/iobuf.h" -#include "butil/sys_byteorder.h" -#include "butil/time.h" -#include "brpc/acceptor.h" -#include "brpc/channel.h" -#include "brpc/controller.h" -#include "brpc/input_messenger.h" -#include "brpc/server.h" -#include "brpc/socket.h" -#include "brpc/errno.pb.h" -#include "brpc/rdma_transport.h" -#include "brpc/rdma/rdma_endpoint.h" -#include "brpc/rdma/rdma_handshake.h" -#include "brpc/rdma/rdma_handshake_constants.h" -#include "brpc/rdma/rdma_handshake.pb.h" -#include "brpc/rdma/rdma_helper.h" -#include "brpc/rdma/rdma_recovery_policy.h" -#include "echo.pb.h" - -using namespace brpc; - -namespace brpc { -namespace rdma { -extern butil::atomic g_rdma_available; -extern bool g_skip_rdma_init; -} // namespace rdma -} // namespace brpc - -static butil::EndPoint g_ep; -static const int64_t WAIT_TIMEOUT_US = 5000000; -static const int64_t WAIT_INTERVAL_US = 1000; - -static bool WaitUntil(const std::function& pred, - int64_t timeout_us = WAIT_TIMEOUT_US) { - const int64_t deadline = butil::gettimeofday_us() + timeout_us; - while (!pred()) { - if (butil::gettimeofday_us() >= deadline) { - return false; - } - usleep((useconds_t)WAIT_INTERVAL_US); - } - return true; -} - -// write(2) may transfer less than asked for, so a test that cares about the -// whole buffer reaching the peer has to loop. -static bool WriteAll(int fd, const void* buf, size_t len) { - const uint8_t* p = (const uint8_t*)buf; - for (size_t done = 0; done < len; ) { - ssize_t n = write(fd, p + done, len - done); - if (n < 0) { - if (errno == EINTR) { - continue; - } - return false; - } - done += n; - } - return true; -} - -static void ConnectToServer(butil::fd_guard* sockfd) { - sockaddr_in addr; - bzero((char*)&addr, sizeof(addr)); - addr.sin_family = AF_INET; - addr.sin_port = htons(g_ep.port); - addr.sin_addr = g_ep.ip; - sockfd->reset(socket(AF_INET, SOCK_STREAM, 0)); - ASSERT_TRUE(*sockfd >= 0); - timeval tv; - tv.tv_sec = WAIT_TIMEOUT_US / 1000000; - tv.tv_usec = WAIT_TIMEOUT_US % 1000000; - ASSERT_EQ(0, setsockopt(*sockfd, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv))); - ASSERT_EQ(0, connect(*sockfd, (sockaddr*)&addr, sizeof(addr))); -} - -class RecoveryEchoService : public ::test::EchoService { - void Echo(google::protobuf::RpcController*, const ::test::EchoRequest*, - ::test::EchoResponse* response, - google::protobuf::Closure* done) override { - ClosureGuard done_guard(done); - response->set_message("MyEchoService"); - } -}; - -class RdmaRuntimeFixture : public ::testing::Test { -protected: - RdmaRuntimeFixture() { - EXPECT_EQ(0, butil::str2endpoint("127.0.0.1:8123", &g_ep)); - EXPECT_EQ(0, _server.AddService(&_service, SERVER_DOESNT_OWN_SERVICE)); - } - - void StartServer(bool use_rdma = true) { - ServerOptions options; - options.enabled_protocols = "baidu_std"; - options.socket_mode = use_rdma ? SOCKET_MODE_RDMA : SOCKET_MODE_TCP; - options.idle_timeout_sec = 5; - ASSERT_EQ(0, _server.Start("127.0.0.1:0", &options)); - g_ep = _server.listen_address(); - } - - void StopServer() { - _server.Stop(0); - _server.Join(); - } - - Socket* GetSocketFromServer(size_t index) { - std::vector sids; - _server._am->ListConnections(&sids); - if (index >= sids.size()) { - return nullptr; - } - SocketUniquePtr socket; - return Socket::Address(sids[index], &socket) == 0 ? socket.get() : nullptr; - } - - Socket* WaitForServerSocket() { - Socket* socket = nullptr; - WaitUntil([this, &socket] { - return (socket = GetSocketFromServer(0)) != nullptr; - }); - return socket; - } - - bool WaitForServerSocketGone() { - return WaitUntil([this] { return GetSocketFromServer(0) == nullptr; }); - } - - Server _server; - RecoveryEchoService _service; -}; - -static RdmaTransport* RdmaTransportOf(Socket* s) { - return static_cast(s->_transport.get()); -} -static RdmaTransport* RdmaTransportOf(const SocketUniquePtr& s) { - return RdmaTransportOf(s.get()); -} - -// Polls until the endpoint reaches `expected` and returns the last state seen, -// so that ASSERT_RDMA_STATE() reports what the endpoint actually settled on. -static rdma::RdmaEndpoint::State WaitForRdmaState( - RdmaTransport* transport, rdma::RdmaEndpoint::State expected) { - rdma::RdmaEndpoint::State state = transport->_rdma_ep->_state; - WaitUntil([transport, expected, &state] { - state = transport->_rdma_ep->_state; - return state == expected; - }); - return state; -} - -// Waits for `transport` to reach `expected`, failing the test if it does not. -#define ASSERT_RDMA_STATE(expected, transport) \ - ASSERT_EQ(expected, WaitForRdmaState(transport, expected)) - -static void MakeV2ClientHello(uint8_t (&data)[rdma::HELLO_V2_MSG_LEN_MIN]) { - rdma::v2_wire::HelloMessage msg{}; - msg.msg_len = rdma::HELLO_V2_MSG_LEN_MIN; - msg.hello_ver = rdma::HELLO_V2_VERSION; - msg.impl_ver = rdma::IMPL_V2_VERSION; - msg.sq_size = 16; - msg.rq_size = 16; - msg.block_size = 8192; - msg.qp_num = 0; - msg.gid = rdma::GetRdmaGid(); - memcpy(data, "RDMA", 4); - msg.Serialize(data + 4); -} - -std::string MakeV3Packet(const rdma::RdmaHello& msg) { - std::string body; - EXPECT_TRUE(msg.SerializeToString(&body)); - std::string packet; - packet.reserve(4 + 4 + body.size()); - packet.append("RDM3", 4); - uint32_t pb_size_be = - butil::HostToNet32(static_cast(body.size())); - packet.append(reinterpret_cast(&pb_size_be), 4); - packet.append(body); - return packet; -} - -// Build a fully-valid RdmaHello: all 6 required fields are set, with -// values that pass RdmaHelloV3Wire::RdmaHelloValid(). -// - block_size = 8192 (>= MIN_BLOCK_SIZE) -// - sq_size / rq_size = 16 (>= MIN_QP_SIZE) -// - gid = exactly 16B (sizeof(ibv_gid)) -// - qp_num = 0 (allowed because g_skip_rdma_init in UT) -rdma::RdmaHello MakeValidV3Hello() { - rdma::RdmaHello msg; - msg.set_block_size(8192); - msg.set_sq_size(16); - msg.set_rq_size(16); - msg.set_lid(0); - ibv_gid gid = rdma::GetRdmaGid(); - msg.set_gid(std::string(reinterpret_cast(gid.raw), - sizeof(gid.raw))); - msg.set_qp_num(0); - return msg; -} - - -class RdmaAdmissionTest : public RdmaRuntimeFixture { -protected: - void SetUp() override { - _saved_mode = rdma::GetRdmaRecoveryMode(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); - } - - void TearDown() override { - StopServer(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(_saved_mode); - } - - void CallEchoOverFallback() { - Channel channel; - ChannelOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.connect_timeout_ms = 1000; - options.timeout_ms = 1000; - options.max_retry = 0; - ASSERT_EQ(0, channel.Init(g_ep, &options)); - Controller cntl; - test::EchoRequest req; - test::EchoResponse res; - req.set_message("runtime recovery"); - test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, nullptr); - ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); - EXPECT_EQ("MyEchoService", res.message()); - SocketUniquePtr s; - ASSERT_EQ(0, Socket::Address(cntl._single_server_id, &s)); - EXPECT_EQ(RdmaTransport::RDMA_OFF, RdmaTransportOf(s)->_rdma_state); - EXPECT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, - RdmaTransportOf(s)->_rdma_ep->_state); - EXPECT_FALSE(s->Failed()); - } - -private: - rdma::RdmaRecoveryMode _saved_mode; -}; - -TEST_F(RdmaAdmissionTest, tcp_only_carries_rpc_to_tcp_server) { - StartServer(false); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -TEST_F(RdmaAdmissionTest, cooled_peer_carries_rpc_over_tcp) { - StartServer(); - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - uint64_t epoch; - ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); - policy.ReportPeerFailure(peer, epoch); - ASSERT_FALSE(policy.AllowNewConnection(peer)); - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -static int PollRnrCompletion(ibv_cq* cq, int, ibv_wc* wc) { - bool* delivered = static_cast(cq->cq_context); - if (*delivered) { - return 0; - } - *delivered = true; - memset(wc, 0, sizeof(*wc)); - wc->status = IBV_WC_RNR_RETRY_EXC_ERR; - return 1; -} - -TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { - StartServer(); - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - SocketOptions cq_options; - cq_options.user = ep; - SocketId cq_sid; - ASSERT_EQ(0, Socket::Create(cq_options, &cq_sid)); - SocketUniquePtr cq_socket; - ASSERT_EQ(0, Socket::Address(cq_sid, &cq_socket)); - - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - uint64_t epoch; - ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); - bool delivered = false; - ibv_context context{}; - context.ops.poll_cq = PollRnrCompletion; - ibv_cq cq{}; - cq.context = &context; - cq.cq_context = &delivered; - rdma::RdmaResource resource; - resource.polling_cq = &cq; - ep->_resource = &resource; - ep->_cq_sid = cq_sid; - ep->_recovery_epoch = epoch; - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - const bool saved_polling = rdma::FLAGS_rdma_use_polling; - rdma::FLAGS_rdma_use_polling = true; - - rdma::RdmaEndpoint::PollCq(cq_socket.get()); - const auto first_report = rdma::GetRdmaRecoveryStatus(); - ep->ReportFailure(rdma::RdmaFailureReason::QP_FATAL); - EXPECT_EQ(first_report.failures[static_cast(rdma::RdmaFailureReason::QP_FATAL)], - rdma::GetRdmaRecoveryStatus().failures[ - static_cast(rdma::RdmaFailureReason::QP_FATAL)]); - EXPECT_TRUE(delivered); - EXPECT_TRUE(socket->Failed()); - EXPECT_FALSE(policy.AllowNewConnection(peer)); - EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); - - // The test owns the mock CQ and resource; ordinary recycling owns neither. - ep->_resource = nullptr; - ep->_cq_sid = INVALID_SOCKET_ID; - resource.polling_cq = nullptr; - cq_socket->_user = nullptr; - cq_socket->SetFailed(); - rdma::FLAGS_rdma_use_polling = saved_polling; - ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); -} - -TEST_F(RdmaAdmissionTest, server_declines_v2_and_v3_when_tcp_only) { - StartServer(); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); - for (int version : {2, 3}) { - butil::fd_guard fd; - ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); - if (version == 2) { - uint8_t hello[rdma::HELLO_V2_MSG_LEN_MIN]; - MakeV2ClientHello(hello); - ASSERT_TRUE(WriteAll(fd, hello, sizeof(hello))); - } else { - const std::string hello = MakeV3Packet(MakeValidV3Hello()); - ASSERT_TRUE(WriteAll(fd, hello.data(), hello.size())); - } - Socket* s = WaitForServerSocket(); - ASSERT_TRUE(s != nullptr); - auto* transport = RdmaTransportOf(s); - ASSERT_RDMA_STATE(rdma::RdmaEndpoint::S_ACK_WAIT, transport); - EXPECT_EQ(RdmaTransport::RDMA_OFF, transport->_rdma_state); - EXPECT_EQ(nullptr, transport->_rdma_ep->_resource); - const uint32_t ack = 0; - ASSERT_TRUE(WriteAll(fd, &ack, sizeof(ack))); - ASSERT_RDMA_STATE(rdma::RdmaEndpoint::FALLBACK_TCP, transport); - EXPECT_FALSE(s->Failed()); - fd.reset(-1); - ASSERT_TRUE(WaitForServerSocketGone()); - } -} - -TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { - EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RETRY_EXC_ERR)); - EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RNR_RETRY_EXC_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_SUCCESS)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_WR_FLUSH_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_LOC_PROT_ERR)); - EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); -} - -TEST_F(RdmaAdmissionTest, async_errors_validate_owner_and_report_once) { - for (ibv_event_type type : {IBV_EVENT_QP_FATAL, IBV_EVENT_CQ_ERR, - IBV_EVENT_QP_ACCESS_ERR}) { - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); - ibv_qp qp{}; - ibv_qp other_qp{}; - ibv_cq cq{}; - ibv_cq other_cq{}; - rdma::RdmaResource resource; - resource.qp = &qp; - resource.polling_cq = &cq; - ep->_resource = &resource; - ep->BindResourceToSocket(); - EXPECT_EQ(sid, reinterpret_cast(qp.qp_context)); - EXPECT_EQ(sid, reinterpret_cast(cq.cq_context)); - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - const auto before = policy.GetStatus(); - ibv_async_event event{}; - event.event_type = type; - if (type == IBV_EVENT_CQ_ERR) { - event.element.cq = &other_cq; - } else { - event.element.qp = &other_qp; - } - rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); - EXPECT_FALSE(socket->Failed()); - if (type == IBV_EVENT_CQ_ERR) { - event.element.cq = &cq; - } else { - event.element.qp = &qp; - } - rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); - EXPECT_TRUE(socket->Failed()); - const auto reason = type == IBV_EVENT_CQ_ERR ? rdma::RdmaFailureReason::CQ_ERROR : - type == IBV_EVENT_QP_FATAL ? rdma::RdmaFailureReason::QP_FATAL : - rdma::RdmaFailureReason::ACCESS_ERROR; - EXPECT_EQ(before.failures[static_cast(reason)] + 1, - policy.GetStatus().failures[static_cast(reason)]); - EXPECT_EQ(type == IBV_EVENT_QP_ACCESS_ERR, policy.AllowNewConnection(peer)); - ep->ReportFailure(rdma::RdmaFailureReason::RNR); - EXPECT_EQ(before.failures[static_cast(rdma::RdmaFailureReason::RNR)], - policy.GetStatus().failures[static_cast(rdma::RdmaFailureReason::RNR)]); - const std::string exposed = bvar::Variable::describe_exposed("rdma_recovery_status"); - EXPECT_NE(std::string::npos, exposed.find("failures")); - EXPECT_NE(std::string::npos, exposed.find("qp_fatal")); - ep->_resource = nullptr; - resource.qp = nullptr; - resource.polling_cq = nullptr; - } -} - -static int FailAckPost(ibv_qp*, ibv_send_wr*, ibv_send_wr**) { - return EIO; -} - -TEST_F(RdmaAdmissionTest, ack_post_error_propagates_and_cools_peer) { - SocketOptions options; - options.socket_mode = SOCKET_MODE_RDMA; - options.remote_side = g_ep; - options.user = get_client_side_messenger(); - SocketId sid; - ASSERT_EQ(0, Socket::Create(options, &sid)); - SocketUniquePtr socket; - ASSERT_EQ(0, Socket::Address(sid, &socket)); - auto* ep = RdmaTransportOf(socket)->_rdma_ep; - auto& policy = rdma::GlobalRdmaRecoveryPolicy(); - const std::string peer = butil::endpoint2str(g_ep).c_str(); - ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); - ibv_context context{}; - context.ops.post_send = FailAckPost; - ibv_qp qp{}; - qp.context = &context; - rdma::RdmaResource resource; - resource.qp = &qp; - ep->_resource = &resource; - ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); - ep->_remote_window_capacity = 0; - ep->_new_rq_wrs.store(1); - ibv_wc wc{}; - wc.opcode = IBV_WC_SEND; - EXPECT_EQ(-1, ep->HandleCompletion(wc)); - EXPECT_EQ(EIO, errno); - EXPECT_FALSE(policy.AllowNewConnection(peer)); - EXPECT_EQ(rdma::RdmaFailureReason::SEND_POST, ep->_runtime_failure.load()); - ep->_resource = nullptr; - resource.qp = nullptr; - socket->SetFailed(); -} - -#endif // BRPC_WITH_RDMA - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); -#if BRPC_WITH_RDMA - rdma::g_rdma_available.store(true, butil::memory_order_relaxed); - rdma::g_skip_rdma_init = true; -#endif - return RUN_ALL_TESTS(); -} diff --git a/test/brpc_rdma_recovery_unittest.cpp b/test/brpc_urma_recovery_unittest.cpp similarity index 63% rename from test/brpc_rdma_recovery_unittest.cpp rename to test/brpc_urma_recovery_unittest.cpp index 734fe794e2..e39bc2aad5 100644 --- a/test/brpc_rdma_recovery_unittest.cpp +++ b/test/brpc_urma_recovery_unittest.cpp @@ -15,38 +15,39 @@ // specific language governing permissions and limitations // under the License. +#if BRPC_WITH_URMA + #include #include #include #include -#include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" namespace brpc { -namespace rdma { +namespace urma { namespace { std::atomic now_us(1000000); int64_t FakeClock() { return now_us.load(); } -class RdmaRecoveryTest : public testing::Test { +class UrmaRecoveryTest : public testing::Test { protected: - RdmaRecoveryTest() : policy(FakeClock, 2) {} + UrmaRecoveryTest() : policy(FakeClock, 2) {} void SetUp() override { now_us = 1000000; } - RdmaRecoveryPolicy policy; + UrmaRecoveryPolicy policy; }; -TEST_F(RdmaRecoveryTest, legacy_preserves_admission) { - policy.ReportPortState(false); +TEST_F(UrmaRecoveryTest, legacy_preserves_admission) { uint64_t epoch; EXPECT_TRUE(policy.AllowNewConnection("a", &epoch)); policy.ReportPeerFailure("a", epoch); EXPECT_TRUE(policy.AllowNewConnection("a")); - EXPECT_TRUE(policy.SetMode(RdmaRecoveryMode::AUTO)); - EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.SetMode(UrmaRecoveryMode::AUTO)); + EXPECT_TRUE(policy.AllowNewConnection("a")); } -TEST_F(RdmaRecoveryTest, peer_failure_only_cools_that_peer) { - policy.SetMode(RdmaRecoveryMode::AUTO); +TEST_F(UrmaRecoveryTest, peer_failure_only_cools_that_peer) { + policy.SetMode(UrmaRecoveryMode::AUTO); policy.SetCooldownMs(10); uint64_t epoch; ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); @@ -60,31 +61,14 @@ TEST_F(RdmaRecoveryTest, peer_failure_only_cools_that_peer) { EXPECT_TRUE(policy.AllowNewConnection("a")); } -TEST_F(RdmaRecoveryTest, port_requires_active_and_elapsed_cooldown) { - policy.SetMode(RdmaRecoveryMode::AUTO); - policy.SetCooldownMs(10); - policy.ReportPortState(false); - EXPECT_FALSE(policy.AllowNewConnection("a")); - EXPECT_FALSE(policy.AllowNewConnection("")); - policy.ReportPortState(true); - EXPECT_FALSE(policy.AllowNewConnection("a")); - now_us += 10000; - EXPECT_TRUE(policy.AllowNewConnection("a")); - policy.ReportPortState(false); - now_us += 10000; - EXPECT_FALSE(policy.AllowNewConnection("a")); - policy.ReportPortState(true); - EXPECT_TRUE(policy.AllowNewConnection("a")); -} - -TEST_F(RdmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { - policy.SetMode(RdmaRecoveryMode::AUTO); +TEST_F(UrmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { + policy.SetMode(UrmaRecoveryMode::AUTO); uint64_t old_epoch; ASSERT_TRUE(policy.AllowNewConnection("a", &old_epoch)); - policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); EXPECT_FALSE(policy.AllowNewConnection("a")); EXPECT_FALSE(policy.AllowNewConnection("")); - policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetMode(UrmaRecoveryMode::AUTO); policy.ReportPeerFailure("a", old_epoch); EXPECT_TRUE(policy.AllowNewConnection("a")); uint64_t epoch; @@ -92,12 +76,12 @@ TEST_F(RdmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { EXPECT_NE(old_epoch, epoch); // Reapplying AUTO does not clear existing quarantine. policy.ReportPeerFailure("a", epoch); - policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetMode(UrmaRecoveryMode::AUTO); EXPECT_FALSE(policy.AllowNewConnection("a")); } -TEST_F(RdmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { - policy.SetMode(RdmaRecoveryMode::AUTO); +TEST_F(UrmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { + policy.SetMode(UrmaRecoveryMode::AUTO); uint64_t epoch; policy.AllowNewConnection("a", &epoch); policy.ReportPeerFailure("a", epoch); @@ -115,29 +99,29 @@ TEST_F(RdmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { EXPECT_FALSE(policy.AllowNewConnection("d")); } -TEST_F(RdmaRecoveryTest, validates_configuration) { - EXPECT_FALSE(policy.SetMode(static_cast(-1))); - EXPECT_EQ(RdmaRecoveryMode::LEGACY, policy.GetMode()); +TEST_F(UrmaRecoveryTest, validates_configuration) { + EXPECT_FALSE(policy.SetMode(static_cast(-1))); + EXPECT_EQ(UrmaRecoveryMode::LEGACY, policy.GetMode()); EXPECT_FALSE(policy.SetCooldownMs(0)); EXPECT_FALSE(policy.SetCooldownMs(-1)); EXPECT_FALSE(policy.SetCooldownMs(static_cast(INT32_MAX) + 1)); EXPECT_TRUE(policy.SetCooldownMs(INT32_MAX)); - policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetMode(UrmaRecoveryMode::AUTO); uint64_t epoch; policy.AllowNewConnection("a", &epoch); policy.ReportPeerFailure("a", epoch); EXPECT_FALSE(policy.AllowNewConnection("a")); } -TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { +TEST_F(UrmaRecoveryTest, concurrent_admission_failure_and_mode_change) { std::vector workers; for (int i = 0; i < 8; ++i) { workers.emplace_back([this, i] { for (int j = 0; j < 200; ++j) { uint64_t epoch; const std::string peer = std::to_string(i); - policy.SetMode(j % 2 ? RdmaRecoveryMode::AUTO : - RdmaRecoveryMode::TCP_ONLY); + policy.SetMode(j % 2 ? UrmaRecoveryMode::AUTO : + UrmaRecoveryMode::TCP_ONLY); policy.AllowNewConnection(peer, &epoch); policy.ReportPeerFailure(peer, epoch); } @@ -146,41 +130,36 @@ TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { for (auto& worker : workers) { worker.join(); } - policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); EXPECT_FALSE(policy.AllowNewConnection("a")); EXPECT_FALSE(policy.AllowNewConnection("")); } -TEST_F(RdmaRecoveryTest, reports_reasons_and_admission_counters) { - policy.SetMode(RdmaRecoveryMode::AUTO); +TEST_F(UrmaRecoveryTest, reports_reasons_and_admission_counters) { + policy.SetMode(UrmaRecoveryMode::AUTO); uint64_t epoch; policy.AllowNewConnection("a", &epoch); policy.RecordNegotiatedConnection(epoch); - policy.ReportPeerFailure("a", epoch, RdmaFailureReason::QP_FATAL); + policy.ReportPeerFailure("a", epoch, UrmaFailureReason::QP_FATAL); EXPECT_FALSE(policy.AllowNewConnection("a")); auto status = policy.GetStatus(); EXPECT_EQ(1u, status.cooled_peers); EXPECT_EQ(1u, status.negotiated_connections); EXPECT_EQ(1u, status.peer_cooldown_admissions); - EXPECT_EQ(1u, status.failures[static_cast(RdmaFailureReason::QP_FATAL)]); - policy.ReportPeerFailure("b", epoch, RdmaFailureReason::ACCESS_ERROR); + EXPECT_EQ(1u, status.failures[static_cast(UrmaFailureReason::QP_FATAL)]); + policy.ReportPeerFailure("b", epoch, UrmaFailureReason::ACCESS_ERROR); EXPECT_TRUE(policy.AllowNewConnection("b")); - policy.ReportPortState(false); - EXPECT_FALSE(policy.AllowNewConnection("b")); - status = policy.GetStatus(); - EXPECT_FALSE(status.port_active); - EXPECT_EQ(5000, status.port_cooldown_ms); - EXPECT_EQ(1u, status.port_errors); - EXPECT_EQ(1u, status.port_cooldown_admissions); now_us += 5000000; EXPECT_EQ(0u, policy.GetStatus().cooled_peers); - policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + policy.SetMode(UrmaRecoveryMode::TCP_ONLY); EXPECT_FALSE(policy.AllowNewConnection("b")); EXPECT_EQ(1u, policy.GetStatus().tcp_only_admissions); - policy.ReportPeerFailure("b", epoch, RdmaFailureReason::CQ_ERROR); - EXPECT_EQ(0u, policy.GetStatus().failures[static_cast(RdmaFailureReason::CQ_ERROR)]); + policy.ReportPeerFailure("b", epoch, UrmaFailureReason::CQ_ERROR); + EXPECT_EQ(0u, policy.GetStatus().failures[static_cast(UrmaFailureReason::CQ_ERROR)]); } } // namespace -} // namespace rdma +} // namespace urma } // namespace brpc + +#endif // BRPC_WITH_URMA diff --git a/test/brpc_urma_runtime_unittest.cpp b/test/brpc_urma_runtime_unittest.cpp index 29081ebb78..eaa027747e 100644 --- a/test/brpc_urma_runtime_unittest.cpp +++ b/test/brpc_urma_runtime_unittest.cpp @@ -366,12 +366,9 @@ class UrmaRecoveryTest : public UrmaRuntimeMockTest { urma::g_skip_urma_init = true; urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); - urma::GlobalUrmaRecoveryPolicy().ReportPortState(true); } void TearDown() override { urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); - rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); UrmaRuntimeMockTest::TearDown(); } void CreateSocket(SocketUniquePtr* socket) { @@ -384,11 +381,9 @@ class UrmaRecoveryTest : public UrmaRuntimeMockTest { } }; -TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { +TEST_F(UrmaRecoveryTest, recovery_mode_and_metrics) { ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); - EXPECT_TRUE(rdma::GlobalRdmaRecoveryPolicy().AllowNewConnection("peer")); - EXPECT_EQ(rdma::RdmaRecoveryMode::LEGACY, rdma::GetRdmaRecoveryMode()); EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); EXPECT_NE(std::string::npos, From 0c5c85defe5954f753a018b0e494b01599de5794 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 2 Nov 2026 01:04:00 -0800 Subject: [PATCH 24/29] Align URMA recovery formatting with brpc review guidance --- example/urma_performance/recovery_client.cpp | 48 +++--- .../urma_performance/send_failure_injector.c | 16 +- .../test_send_failure_injector.c | 18 +- src/brpc/urma/urma_endpoint.cpp | 98 ++++++----- src/brpc/urma/urma_helper.cpp | 21 +-- src/brpc/urma/urma_helper.h | 2 +- src/brpc/urma/urma_recovery_policy.cpp | 74 +++++---- src/brpc/urma/urma_recovery_policy.h | 20 ++- test/brpc_urma_copy_unittest.cpp | 16 +- test/brpc_urma_recovery_unittest.cpp | 11 +- test/brpc_urma_runtime_unittest.cpp | 157 +++++++++++------- test/brpc_urma_transport_unittest.cpp | 22 ++- 12 files changed, 308 insertions(+), 195 deletions(-) diff --git a/example/urma_performance/recovery_client.cpp b/example/urma_performance/recovery_client.cpp index 5c6fc9ed5f..a1a53fe676 100644 --- a/example/urma_performance/recovery_client.cpp +++ b/example/urma_performance/recovery_client.cpp @@ -29,15 +29,18 @@ DEFINE_string(server, "127.0.0.1:8003", "Performance test server address"); DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); -DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); -DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); +DEFINE_string(mode_file, "", + "Read a recovery mode from this file while running"); +DEFINE_string(connection_type, "short", + "short exercises new admission on each RPC"); DEFINE_string(transport, "urma", "urma or tcp"); DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); DEFINE_int32(interval_ms, 100, "Pause between RPCs"); DEFINE_int32(timeout_ms, 1000, "RPC deadline"); DEFINE_int32(cooldown_ms, 5000, "Path cooldown"); DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); -DEFINE_int32(attachment_size, 0, "Echo attachment bytes and verify the returned content"); +DEFINE_int32(attachment_size, 0, + "Echo attachment bytes and verify the returned content"); static bool SetMode(const std::string& value) { using brpc::urma::UrmaRecoveryMode; @@ -60,18 +63,18 @@ int main(int argc, char* argv[]) { std::cerr << "Invalid transport\n"; return 1; } - const bool cooldown_ok = brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms); + const bool cooldown_ok = + brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms); if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || - FLAGS_attachment_size < 0 || - !cooldown_ok || - !SetMode(FLAGS_mode)) { - std::cerr << "Invalid duration, interval, timeout, cooldown, mode or attachment size\n"; + FLAGS_attachment_size < 0 || !cooldown_ok || !SetMode(FLAGS_mode)) { + std::cerr + << "Invalid duration, interval, timeout, cooldown, mode or attachment size\n"; return 1; } brpc::Channel channel; brpc::ChannelOptions options; - options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA : - brpc::SOCKET_MODE_TCP; + options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA + : brpc::SOCKET_MODE_TCP; options.connection_type = FLAGS_connection_type; options.timeout_ms = FLAGS_timeout_ms; options.connect_timeout_ms = FLAGS_timeout_ms; @@ -92,10 +95,12 @@ int main(int argc, char* argv[]) { } const int64_t start = butil::monotonic_time_us(); int64_t next_report = start; - std::cout << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," - "cooled_peers\n"; + std::cout + << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," + "cooled_peers\n"; while (!brpc::IsAskedToQuit() && - butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { + butil::monotonic_time_us() - start < + static_cast(FLAGS_seconds) * 1000000) { if (!FLAGS_mode_file.empty()) { std::ifstream input(FLAGS_mode_file); std::string value; @@ -103,7 +108,8 @@ int main(int argc, char* argv[]) { if (SetMode(value)) { mode = value; } else { - std::cerr << "Ignoring invalid recovery mode: " << value << '\n'; + std::cerr << "Ignoring invalid recovery mode: " << value + << '\n'; } } } @@ -116,23 +122,27 @@ int main(int argc, char* argv[]) { payload[i] = static_cast(sequence >> (i * 8)); } cntl.request_attachment().append(payload); - test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, nullptr); + test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, + nullptr); if (cntl.Failed()) { ++failed; std::cerr << "RPC failure: " << cntl.ErrorText() << '\n'; - } else if (!payload.empty() && cntl.response_attachment().to_string() != payload) { + } else if (!payload.empty() && + cntl.response_attachment().to_string() != payload) { ++failed; ++validation_failed; std::cerr << "Attachment mismatch: expected=" << payload.size() - << " actual=" << cntl.response_attachment().size() << '\n'; + << " actual=" << cntl.response_attachment().size() + << '\n'; } else { ++succeeded; } const int64_t now = butil::monotonic_time_us(); if (now >= next_report) { const auto s = brpc::urma::GetUrmaRecoveryStatus(); - std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded - << ',' << failed << ',' << s.negotiated_connections << ',' + std::cout << (now - start) / 1000000 << ',' << mode << ',' + << succeeded << ',' << failed << ',' + << s.negotiated_connections << ',' << s.tcp_only_admissions + s.peer_cooldown_admissions << ',' << s.cooled_peers << std::endl; next_report = now + 1000000; diff --git a/example/urma_performance/send_failure_injector.c b/example/urma_performance/send_failure_injector.c index 321fa5c85a..60b028a2af 100644 --- a/example/urma_performance/send_failure_injector.c +++ b/example/urma_performance/send_failure_injector.c @@ -25,7 +25,8 @@ #include #include "urma_api.h" -typedef urma_status_t (*PostSend)(urma_jetty_t*, urma_jfs_wr_t*, urma_jfs_wr_t**); +typedef urma_status_t (*PostSend)(urma_jetty_t*, urma_jfs_wr_t*, + urma_jfs_wr_t**); static PostSend g_post_send; static pthread_once_t g_resolve_once = PTHREAD_ONCE_INIT; static atomic_bool g_injected = ATOMIC_VAR_INIT(0); @@ -33,7 +34,8 @@ static atomic_bool g_injected = ATOMIC_VAR_INIT(0); static void ResolvePostSend(void) { g_post_send = (PostSend)dlsym(RTLD_NEXT, "urma_post_jetty_send_wr"); if (!g_post_send) { - fprintf(stderr, "URMA injector: cannot resolve real post_send: %s\n", dlerror()); + fprintf(stderr, "URMA injector: cannot resolve real post_send: %s\n", + dlerror()); } } @@ -41,7 +43,7 @@ static void ResolvePostSend(void) { // by BRPC_URMA_FAIL_ONCE_FILE rejects one data SEND before it reaches liburma. // Credit ACKs and all subsequent sends are forwarded to the provider. urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, - urma_jfs_wr_t** bad_wr) { + urma_jfs_wr_t** bad_wr) { pthread_once(&g_resolve_once, ResolvePostSend); if (!g_post_send) { if (bad_wr) { @@ -54,12 +56,16 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, if (wr && wr->opcode == URMA_OPC_SEND && marker && marker[0] && access(marker, F_OK) == 0 && !atomic_exchange(&g_injected, 1)) { if (unlink(marker) != 0) { - fprintf(stderr, "URMA injector: cannot remove marker: %s\n", marker); + fprintf(stderr, "URMA injector: cannot remove marker: %s\n", + marker); } if (bad_wr) { *bad_wr = wr; } - fprintf(stderr, "URMA injector: injected one data SEND post failure, errno=%d\n", EIO); + fprintf( + stderr, + "URMA injector: injected one data SEND post failure, errno=%d\n", + EIO); errno = EIO; return EIO; } diff --git a/example/urma_performance/test_send_failure_injector.c b/example/urma_performance/test_send_failure_injector.c index 42269c90b4..095ef0a6ff 100644 --- a/example/urma_performance/test_send_failure_injector.c +++ b/example/urma_performance/test_send_failure_injector.c @@ -28,7 +28,7 @@ static int g_posts; int UrmaInjectorTestPostCount(void) { return g_posts; } urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, - urma_jfs_wr_t** bad_wr) { + urma_jfs_wr_t** bad_wr) { (void)jetty; (void)wr; if (bad_wr) { @@ -41,12 +41,13 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t* jetty, urma_jfs_wr_t* wr, #else int UrmaInjectorTestPostCount(void); -#define CHECK(expr) do { \ - if (!(expr)) { \ - fprintf(stderr, "Check failed: %s at line %d\n", #expr, __LINE__); \ - return 1; \ - } \ -} while (0) +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + fprintf(stderr, "Check failed: %s at line %d\n", #expr, __LINE__); \ + return 1; \ + } \ + } while (0) int main(void) { char marker[] = "/tmp/brpc-urma-injector-XXXXXX"; @@ -83,7 +84,8 @@ int main(void) { CHECK(urma_post_jetty_send_wr(NULL, &wr, &bad) == URMA_SUCCESS); CHECK(UrmaInjectorTestPostCount() == 4); CHECK(unlink(marker) == 0); - puts("PASS: forwarding, credit ACK, single failure, and resumed forwarding"); + puts( + "PASS: forwarding, credit ACK, single failure, and resumed forwarding"); return 0; } diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index d3290dcb3a..f8da944040 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -170,7 +170,8 @@ void UrmaEndpoint::Reset() { DeallocateResources(); _state = UNINIT; _recovery_epoch = 0; - _runtime_failure.store(UrmaFailureReason::NONE, butil::memory_order_relaxed); + _runtime_failure.store(UrmaFailureReason::NONE, + butil::memory_order_relaxed); _handshake_version = 0; _remote_recv_block_size = 0; _local_window_capacity = 0; @@ -667,13 +668,15 @@ class UrmaIOBuf : private butil::IOBuf { this_len = max_len - len; } if (!InitUrmaLocalSge(&sglist[*sge_index], - reinterpret_cast(start), this_len, tseg)) { + reinterpret_cast(start), this_len, + tseg)) { LOG(ERROR) << "Invalid URMA send SGE: address=" << reinterpret_cast(start) << " length=" << this_len << " segment_va=" << tseg->seg.ubva.va << " segment_len=" << tseg->seg.len - << " context_matches=" << (tseg->urma_ctx == GetUrmaContext()); + << " context_matches=" + << (tseg->urma_ctx == GetUrmaContext()); return -1; } cutn(to, this_len); @@ -722,7 +725,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { : GetUrmaRecvBlockSize(); if (copy_send) { max_len = std::min(max_len, GetUrmaRecvBlockSize() - - (URMA_SEND_BUFFER_ALIGNMENT - 1)); + (URMA_SEND_BUFFER_ALIGNMENT - 1)); } while (sge_index < static_cast(max_sge) && this_len < max_len && current < ndata) { @@ -731,10 +734,11 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ++current; continue; } - ssize_t n = copy_send ? - static_cast(data->cutn(to, max_len - this_len)) : - data->cut_into_sglist(sglist, &sge_index, to, - max_sge, max_len - this_len); + ssize_t n = + copy_send + ? static_cast(data->cutn(to, max_len - this_len)) + : data->cut_into_sglist(sglist, &sge_index, to, max_sge, + max_len - this_len); if (n < 0) { if (errno == ERDMAMEM) { ReportFailure(UrmaFailureReason::ACCESS_ERROR); @@ -748,8 +752,9 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { } if (copy_send) { if (!CopyUrmaSendBuffer(to, &sglist[0])) { - LOG(ERROR) << "Fail to prepare aligned URMA send buffer: length=" - << this_len << " errno=" << errno; + LOG(ERROR) + << "Fail to prepare aligned URMA send buffer: length=" + << this_len << " errno=" << errno; ReportFailure(UrmaFailureReason::ACCESS_ERROR); return -1; } @@ -966,38 +971,37 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { LOG(WARNING) << "URMA completion failed, status=" << cr.status << " direction=" << (cr.flag.bs.s_r ? "recv" : "send") << " flags=" << static_cast(cr.flag.value) - << " opcode=" << cr.opcode - << " local_id=" << cr.local_id - << " tpn=" << cr.tpn - << " user_ctx=" << cr.user_ctx - << " completion_len=" << cr.completion_len - << " jetty_id=" - << (_resource && _resource->jetty ? - _resource->jetty->jetty_id.id : 0) + << " opcode=" << cr.opcode << " local_id=" << cr.local_id + << " tpn=" << cr.tpn << " user_ctx=" << cr.user_ctx + << " completion_len=" << cr.completion_len << " jetty_id=" + << (_resource && _resource->jetty + ? _resource->jetty->jetty_id.id + : 0) << " jfr_id=" - << (_resource && _resource->jfr ? - _resource->jfr->jfr_id.id : 0) - << " state=" << GetStateStr() - << " on " << _socket->description(); + << (_resource && _resource->jfr ? _resource->jfr->jfr_id.id + : 0) + << " state=" << GetStateStr() << " on " + << _socket->description(); if (!cr.flag.bs.s_r && cr.user_ctx > 0 && cr.user_ctx <= _sbuf.size()) { const size_t slot = static_cast(cr.user_ctx - 1); const auto& buffer = _sbuf[slot]; LOG(WARNING) << "URMA failed SEND buffer: slot=" << slot << " bytes=" << buffer.size() - << " blocks=" << buffer.backing_block_num() - << " on " << _socket->description(); + << " blocks=" << buffer.backing_block_num() << " on " + << _socket->description(); for (size_t i = 0; i < buffer.backing_block_num(); ++i) { const auto block = buffer.backing_block(i); auto* segment = GetPoolSegFor(const_cast(block.data())); - LOG(WARNING) << "URMA failed SEND block: slot=" << slot - << " block=" << i - << " address=" << reinterpret_cast(block.data()) - << " length=" << block.size() - << " pool_segment=" << (segment != nullptr) - << " segment_va=" << (segment ? segment->seg.ubva.va : 0) - << " segment_len=" << (segment ? segment->seg.len : 0) - << " segment_token_id=" << (segment ? segment->seg.token_id : 0) - << " on " << _socket->description(); + LOG(WARNING) + << "URMA failed SEND block: slot=" << slot << " block=" << i + << " address=" << reinterpret_cast(block.data()) + << " length=" << block.size() + << " pool_segment=" << (segment != nullptr) + << " segment_va=" << (segment ? segment->seg.ubva.va : 0) + << " segment_len=" << (segment ? segment->seg.len : 0) + << " segment_token_id=" + << (segment ? segment->seg.token_id : 0) << " on " + << _socket->description(); } } switch (cr.status) { @@ -1415,8 +1419,9 @@ void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { return; } GlobalUrmaRecoveryPolicy().ReportPeerFailure( - _socket->CreatedByConnect() ? - butil::endpoint2str(_socket->remote_side()).c_str() : "", + _socket->CreatedByConnect() + ? butil::endpoint2str(_socket->remote_side()).c_str() + : "", _recovery_epoch, reason); } @@ -1446,9 +1451,9 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { SocketUniquePtr s(ep->_socket); auto* tp = static_cast(s->_transport.get()); UrmaConnect::RunGuard guard(static_cast(s->_app_connect.get())); - if (!IsUrmaAvailable() || - !GlobalUrmaRecoveryPolicy().AllowNewConnection( - butil::endpoint2str(s->remote_side()).c_str(), &ep->_recovery_epoch)) { + if (!IsUrmaAvailable() || !GlobalUrmaRecoveryPolicy().AllowNewConnection( + butil::endpoint2str(s->remote_side()).c_str(), + &ep->_recovery_epoch)) { ep->FallbackToTcp(tp, true); return nullptr; } @@ -1537,11 +1542,12 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->FailHandshake(tp, EPROTO, "invalid client hello"); return nullptr; } - if (!IsUrmaAvailable() || - !GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch)) { + if (!IsUrmaAvailable() || !GlobalUrmaRecoveryPolicy().AllowNewConnection( + "", &ep->_recovery_epoch)) { tp->_urma_state = UrmaTransport::URMA_OFF; if (hs->SendLocalHello() < 0) { - ep->FailHandshake(tp, errno ? errno : EIO, "send TCP fallback hello"); + ep->FailHandshake(tp, errno ? errno : EIO, + "send TCP fallback hello"); return nullptr; } ep->FallbackToTcp(tp, true); @@ -1597,7 +1603,8 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; - GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection( + ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); ep->DrainEstablishedTcp(); } else { @@ -1684,11 +1691,12 @@ std::string UrmaEndpoint::GetStateStr() const { } void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { - os << "state=" << GetStateStr() - << " sq_size=" << _sq_size << " rq_size=" << _rq_size + os << "state=" << GetStateStr() << " sq_size=" << _sq_size + << " rq_size=" << _rq_size << " remote_recv_block_size=" << _remote_recv_block_size << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) - << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " remote_rq_window=" + << _remote_rq_window_size.load(butil::memory_order_relaxed) << " recovery_epoch=" << _recovery_epoch << " runtime_failure=" << UrmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index b9aa4b84c0..47d9193f4c 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -434,9 +434,8 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, } const uint64_t base = segment->seg.ubva.va; const uint64_t size = segment->seg.len; - if (size > std::numeric_limits::max() - base || - address < base || address - base > size || - length > size - (address - base)) { + if (size > std::numeric_limits::max() - base || address < base || + address - base > size || length > size - (address - base)) { errno = ERDMAMEM; return false; } @@ -445,7 +444,7 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, } bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, - urma_target_seg_t* (*lookup)(void*)) { + urma_target_seg_t* (*lookup)(void*)) { const size_t length = buffer->size(); const size_t capacity = GetUrmaRecvBlockSize(); if (length == 0 || capacity < URMA_SEND_BUFFER_ALIGNMENT || @@ -464,9 +463,10 @@ bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, errno = ENOMEM; return false; } - padding = (URMA_SEND_BUFFER_ALIGNMENT - - reinterpret_cast(data) % URMA_SEND_BUFFER_ALIGNMENT) % - URMA_SEND_BUFFER_ALIGNMENT; + padding = + (URMA_SEND_BUFFER_ALIGNMENT - + reinterpret_cast(data) % URMA_SEND_BUFFER_ALIGNMENT) % + URMA_SEND_BUFFER_ALIGNMENT; if (padding + length > static_cast(available)) { errno = ENOMEM; return false; @@ -476,7 +476,8 @@ bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, stream.BackUp(available - static_cast(padding + length)); } copied.pop_front(padding); - if (!InitUrmaLocalSge(sge, reinterpret_cast(data), length, lookup(data))) { + if (!InitUrmaLocalSge(sge, reinterpret_cast(data), length, + lookup(data))) { return false; } buffer->swap(copied); @@ -597,8 +598,8 @@ static bool GlobalUrmaInitializeImpl() { g_device = nullptr; return false; } - if (!SelectUrmaTransportPath(g_device_attr, FLAGS_urma_tp_type, - &g_tp_type, &g_jetty_priority)) { + if (!SelectUrmaTransportPath(g_device_attr, FLAGS_urma_tp_type, &g_tp_type, + &g_jetty_priority)) { LOG(ERROR) << "URMA path selection failed: requested=" << FLAGS_urma_tp_type << " device=" << device_name << " ctp_priority=" diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index b46475f3da..5cebf6cfdd 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -101,7 +101,7 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, constexpr size_t URMA_SEND_BUFFER_ALIGNMENT = 64; // Copy host payloads into one aligned block and transfer its ownership to buffer. bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, - urma_target_seg_t* (*lookup)(void*) = GetPoolSegFor); + urma_target_seg_t* (*lookup)(void*) = GetPoolSegFor); // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/src/brpc/urma/urma_recovery_policy.cpp b/src/brpc/urma/urma_recovery_policy.cpp index 157ae02a79..73bb6eda3a 100644 --- a/src/brpc/urma/urma_recovery_policy.cpp +++ b/src/brpc/urma/urma_recovery_policy.cpp @@ -24,19 +24,32 @@ namespace urma { const char* UrmaFailureReasonName(UrmaFailureReason reason) { switch (reason) { - case UrmaFailureReason::NONE: return "none"; - case UrmaFailureReason::RETRY_EXHAUSTED: return "retry_exhausted"; - case UrmaFailureReason::RNR: return "rnr"; - case UrmaFailureReason::QP_FATAL: return "qp_fatal"; - case UrmaFailureReason::CQ_ERROR: return "cq_error"; - case UrmaFailureReason::CQ_POLL: return "cq_poll"; - case UrmaFailureReason::CQ_EVENT: return "cq_event"; - case UrmaFailureReason::CQ_NOTIFY: return "cq_notify"; - case UrmaFailureReason::SEND_POST: return "send_post"; - case UrmaFailureReason::ACCESS_ERROR: return "access_error"; - case UrmaFailureReason::TRANSPORT_ERROR: return "transport_error"; - case UrmaFailureReason::RECV_POST: return "recv_post"; - default: return "unknown"; + case UrmaFailureReason::NONE: + return "none"; + case UrmaFailureReason::RETRY_EXHAUSTED: + return "retry_exhausted"; + case UrmaFailureReason::RNR: + return "rnr"; + case UrmaFailureReason::QP_FATAL: + return "qp_fatal"; + case UrmaFailureReason::CQ_ERROR: + return "cq_error"; + case UrmaFailureReason::CQ_POLL: + return "cq_poll"; + case UrmaFailureReason::CQ_EVENT: + return "cq_event"; + case UrmaFailureReason::CQ_NOTIFY: + return "cq_notify"; + case UrmaFailureReason::SEND_POST: + return "send_post"; + case UrmaFailureReason::ACCESS_ERROR: + return "access_error"; + case UrmaFailureReason::TRANSPORT_ERROR: + return "transport_error"; + case UrmaFailureReason::RECV_POST: + return "recv_post"; + default: + return "unknown"; } } @@ -45,8 +58,7 @@ UrmaRecoveryPolicy::UrmaRecoveryPolicy(Clock clock, size_t max_peers) , _max_peers(max_peers) , _mode(UrmaRecoveryMode::LEGACY) , _epoch(1) - , _cooldown_us(5000000) { -} + , _cooldown_us(5000000) {} bool UrmaRecoveryPolicy::SetMode(UrmaRecoveryMode mode) { if (mode != UrmaRecoveryMode::LEGACY && mode != UrmaRecoveryMode::AUTO && @@ -77,7 +89,7 @@ bool UrmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { } bool UrmaRecoveryPolicy::AllowNewConnection(const std::string& peer, - uint64_t* epoch) { + uint64_t* epoch) { std::lock_guard lock(_mutex); if (epoch != nullptr) { *epoch = _epoch; @@ -103,8 +115,8 @@ bool UrmaRecoveryPolicy::AllowNewConnection(const std::string& peer, } void UrmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, - uint64_t epoch, - UrmaFailureReason reason) { + uint64_t epoch, + UrmaFailureReason reason) { std::lock_guard lock(_mutex); if (epoch != _epoch || reason <= UrmaFailureReason::NONE || reason >= UrmaFailureReason::COUNT) { @@ -158,10 +170,10 @@ void UrmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { static void DescribeRecovery(std::ostream& os, void* arg) { const auto s = static_cast(arg)->GetStatus(); os << "{\"mode\":\"" - << (s.mode == UrmaRecoveryMode::AUTO ? "auto" : - s.mode == UrmaRecoveryMode::TCP_ONLY ? "tcp_only" : "legacy") - << "\",\"epoch\":" << s.epoch - << ",\"cooled_peers\":" << s.cooled_peers + << (s.mode == UrmaRecoveryMode::AUTO ? "auto" + : s.mode == UrmaRecoveryMode::TCP_ONLY ? "tcp_only" + : "legacy") + << "\",\"epoch\":" << s.epoch << ",\"cooled_peers\":" << s.cooled_peers << ",\"tcp_only_admissions\":" << s.tcp_only_admissions << ",\"peer_cooldown_admissions\":" << s.peer_cooldown_admissions << ",\"cache_evictions\":" << s.cache_evictions @@ -181,11 +193,14 @@ struct RecoveryMetrics { RecoveryMetrics(UrmaRecoveryPolicy* policy, const char* status_name, const char* fallback_name) : status(status_name, DescribeRecovery, policy) - , fallbacks(fallback_name, [](void* arg) { - const auto s = static_cast(arg)->GetStatus(); - return s.tcp_only_admissions + s.peer_cooldown_admissions; - }, policy) { - } + , fallbacks( + fallback_name, + [](void* arg) { + const auto s = + static_cast(arg)->GetStatus(); + return s.tcp_only_admissions + s.peer_cooldown_admissions; + }, + policy) {} bvar::PassiveStatus status; bvar::PassiveStatus fallbacks; }; @@ -193,8 +208,9 @@ struct RecoveryMetrics { UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy() { static UrmaRecoveryPolicy* policy = new UrmaRecoveryPolicy(butil::monotonic_time_us); - static RecoveryMetrics* metrics = new RecoveryMetrics( - policy, "urma_recovery_status", "urma_recovery_tcp_fallback_admissions"); + static RecoveryMetrics* metrics = + new RecoveryMetrics(policy, "urma_recovery_status", + "urma_recovery_tcp_fallback_admissions"); (void)metrics; return *policy; } diff --git a/src/brpc/urma/urma_recovery_policy.h b/src/brpc/urma/urma_recovery_policy.h index e9cbbf5005..101b629de8 100644 --- a/src/brpc/urma/urma_recovery_policy.h +++ b/src/brpc/urma/urma_recovery_policy.h @@ -30,8 +30,19 @@ namespace urma { enum class UrmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; enum class UrmaFailureReason { - NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, - CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, TRANSPORT_ERROR, RECV_POST, COUNT + NONE, + RETRY_EXHAUSTED, + RNR, + QP_FATAL, + CQ_ERROR, + CQ_POLL, + CQ_EVENT, + CQ_NOTIFY, + SEND_POST, + ACCESS_ERROR, + TRANSPORT_ERROR, + RECV_POST, + COUNT }; const char* UrmaFailureReasonName(UrmaFailureReason reason); @@ -58,8 +69,9 @@ class UrmaRecoveryPolicy { // Positive milliseconds, at most INT32_MAX. Applies to future failures. bool SetCooldownMs(int64_t cooldown_ms); bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); - void ReportPeerFailure(const std::string& peer, uint64_t epoch, - UrmaFailureReason reason = UrmaFailureReason::RETRY_EXHAUSTED); + void ReportPeerFailure( + const std::string& peer, uint64_t epoch, + UrmaFailureReason reason = UrmaFailureReason::RETRY_EXHAUSTED); UrmaRecoveryStatus GetStatus(); void RecordNegotiatedConnection(uint64_t epoch); diff --git a/test/brpc_urma_copy_unittest.cpp b/test/brpc_urma_copy_unittest.cpp index da403f608a..fbb61d5a97 100644 --- a/test/brpc_urma_copy_unittest.cpp +++ b/test/brpc_urma_copy_unittest.cpp @@ -41,8 +41,9 @@ DECLARE_int32(urma_client_handshake_version); namespace { class CopiedEchoService : public test::EchoService { - void Echo(google::protobuf::RpcController* base, const test::EchoRequest* request, - test::EchoResponse* response, google::protobuf::Closure* done) override { + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { brpc::ClosureGuard guard(done); auto* cntl = static_cast(base); response->set_message(request->message()); @@ -61,7 +62,7 @@ class UrmaCopiedRpcTest : public ::testing::TestWithParam { int count = 0; auto** devices = urma_get_device_list(&count); _mock = devices && count > 0 && - strcmp(devices[0]->name, "mock_urma_device") == 0; + strcmp(devices[0]->name, "mock_urma_device") == 0; urma_free_device_list(devices); if (status == URMA_SUCCESS) { urma_uninit(); @@ -87,7 +88,8 @@ class UrmaCopiedRpcTest : public ::testing::TestWithParam { }; bool UrmaCopiedRpcTest::_mock = false; -TEST_P(UrmaCopiedRpcTest, copies_payload_and_attachment_across_wrs_and_credit_returns) { +TEST_P(UrmaCopiedRpcTest, + copies_payload_and_attachment_across_wrs_and_credit_returns) { CopiedEchoService service; brpc::Server server; ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); @@ -111,7 +113,8 @@ TEST_P(UrmaCopiedRpcTest, copies_payload_and_attachment_across_wrs_and_credit_re test::EchoResponse response; request.set_message(payload); cntl.request_attachment().append(payload); - test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); EXPECT_EQ(payload, response.message()); EXPECT_EQ(payload, cntl.response_attachment().to_string()); @@ -122,7 +125,8 @@ TEST_P(UrmaCopiedRpcTest, copies_payload_and_attachment_across_wrs_and_credit_re server.Join(); } -INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaCopiedRpcTest, ::testing::Values(2, 3)); +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaCopiedRpcTest, + ::testing::Values(2, 3)); } // namespace #endif // BRPC_WITH_URMA diff --git a/test/brpc_urma_recovery_unittest.cpp b/test/brpc_urma_recovery_unittest.cpp index e39bc2aad5..afb996b154 100644 --- a/test/brpc_urma_recovery_unittest.cpp +++ b/test/brpc_urma_recovery_unittest.cpp @@ -120,8 +120,8 @@ TEST_F(UrmaRecoveryTest, concurrent_admission_failure_and_mode_change) { for (int j = 0; j < 200; ++j) { uint64_t epoch; const std::string peer = std::to_string(i); - policy.SetMode(j % 2 ? UrmaRecoveryMode::AUTO : - UrmaRecoveryMode::TCP_ONLY); + policy.SetMode(j % 2 ? UrmaRecoveryMode::AUTO + : UrmaRecoveryMode::TCP_ONLY); policy.AllowNewConnection(peer, &epoch); policy.ReportPeerFailure(peer, epoch); } @@ -146,7 +146,8 @@ TEST_F(UrmaRecoveryTest, reports_reasons_and_admission_counters) { EXPECT_EQ(1u, status.cooled_peers); EXPECT_EQ(1u, status.negotiated_connections); EXPECT_EQ(1u, status.peer_cooldown_admissions); - EXPECT_EQ(1u, status.failures[static_cast(UrmaFailureReason::QP_FATAL)]); + EXPECT_EQ( + 1u, status.failures[static_cast(UrmaFailureReason::QP_FATAL)]); policy.ReportPeerFailure("b", epoch, UrmaFailureReason::ACCESS_ERROR); EXPECT_TRUE(policy.AllowNewConnection("b")); now_us += 5000000; @@ -155,7 +156,9 @@ TEST_F(UrmaRecoveryTest, reports_reasons_and_admission_counters) { EXPECT_FALSE(policy.AllowNewConnection("b")); EXPECT_EQ(1u, policy.GetStatus().tcp_only_admissions); policy.ReportPeerFailure("b", epoch, UrmaFailureReason::CQ_ERROR); - EXPECT_EQ(0u, policy.GetStatus().failures[static_cast(UrmaFailureReason::CQ_ERROR)]); + EXPECT_EQ(0u, + policy.GetStatus() + .failures[static_cast(UrmaFailureReason::CQ_ERROR)]); } } // namespace diff --git a/test/brpc_urma_runtime_unittest.cpp b/test/brpc_urma_runtime_unittest.cpp index eaa027747e..fac5b91fbf 100644 --- a/test/brpc_urma_runtime_unittest.cpp +++ b/test/brpc_urma_runtime_unittest.cpp @@ -62,7 +62,8 @@ TEST(UrmaRuntimeHelperTest, combined_priority_and_invalid_transport_type) { attr, static_cast(255))); } -TEST(UrmaRuntimeHelperTest, auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { +TEST(UrmaRuntimeHelperTest, + auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_devices) { urma_device_attr_t attr{}; attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; attr.dev_cap.priority_info[0].SL = 2; @@ -77,7 +78,8 @@ TEST(UrmaRuntimeHelperTest, auto_path_prefers_ctp_and_selects_rtp_on_rtp_only_de EXPECT_EQ(7, priority); } -TEST(UrmaRuntimeHelperTest, explicit_path_uses_matching_priority_and_combined_capabilities) { +TEST(UrmaRuntimeHelperTest, + explicit_path_uses_matching_priority_and_combined_capabilities) { urma_device_attr_t attr{}; attr.dev_cap.priority_info[2].tp_type.bs.rtp = 1; attr.dev_cap.priority_info[2].tp_type.bs.ctp = 1; @@ -85,13 +87,15 @@ TEST(UrmaRuntimeHelperTest, explicit_path_uses_matching_priority_and_combined_ca urma_tp_type_t path = URMA_UTP; uint8_t priority = 15; for (const auto& requested : {"rtp", "ctp"}) { - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + ASSERT_TRUE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); EXPECT_EQ(std::string(requested) == "rtp" ? URMA_RTP : URMA_CTP, path); EXPECT_EQ(2, priority); } } -TEST(UrmaRuntimeHelperTest, rejects_unsupported_paths_in_a_reported_capability_table) { +TEST(UrmaRuntimeHelperTest, + rejects_unsupported_paths_in_a_reported_capability_table) { urma_device_attr_t attr{}; attr.dev_cap.priority_info[0].tp_type.bs.rtp = 1; urma_tp_type_t path = URMA_UTP; @@ -105,12 +109,14 @@ TEST(UrmaRuntimeHelperTest, rejects_unsupported_paths_in_a_reported_capability_t EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "rtp", &path, &priority)); } -TEST(UrmaRuntimeHelperTest, providers_without_path_capabilities_keep_compatibility_priorities) { +TEST(UrmaRuntimeHelperTest, + providers_without_path_capabilities_keep_compatibility_priorities) { urma_device_attr_t attr{}; urma_tp_type_t path = URMA_UTP; uint8_t priority = 15; for (const auto& requested : {"auto", "ctp", "rtp"}) { - ASSERT_TRUE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + ASSERT_TRUE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); const bool rtp = std::string(requested) == "rtp"; EXPECT_EQ(rtp ? URMA_RTP : URMA_CTP, path); EXPECT_EQ(rtp ? 0 : 6, priority); @@ -122,13 +128,16 @@ TEST(UrmaRuntimeHelperTest, rejects_invalid_path_configuration) { urma_tp_type_t path = URMA_CTP; uint8_t priority = 6; for (const auto& requested : {"", "utp", "RM", "typo"}) { - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, requested, &path, &priority)); } - EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); + EXPECT_FALSE( + urma::SelectUrmaTransportPath(attr, "auto", nullptr, &priority)); EXPECT_FALSE(urma::SelectUrmaTransportPath(attr, "auto", &path, nullptr)); } -TEST(UrmaRuntimeHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { +TEST(UrmaRuntimeHelperTest, + local_sge_initializes_optional_fields_and_accepts_segment_edges) { urma_target_seg_t segment{}; segment.urma_ctx = urma::GetUrmaContext(); segment.seg.ubva.va = 0x1000; @@ -152,9 +161,12 @@ TEST(UrmaRuntimeHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { segment.seg.len = 8192; urma_sge_t sge{}; for (auto values : {std::pair(0xfff, 1), - {0x3000, 1}, {0x2fff, 2}, {0x1000, 0}, + {0x3000, 1}, + {0x2fff, 2}, + {0x1000, 0}, {0x1000, static_cast(UINT32_MAX) + 1}}) { - EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, &segment)); + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, + &segment)); EXPECT_EQ(ERDMAMEM, errno); } EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); @@ -177,16 +189,15 @@ static urma_target_seg_t* ResolveCopyTestSegment(void* address) { return &g_copy_test_segment; } -static urma_target_seg_t* RejectCopyTestSegment(void*) { - return nullptr; -} +static urma_target_seg_t* RejectCopyTestSegment(void*) { return nullptr; } static void DeleteCopyTestData(void* data) { ++g_copy_test_releases; delete[] static_cast(data); } -TEST(UrmaRuntimeHelperTest, copied_send_coalesces_host_fragments_and_retains_aligned_payload) { +TEST(UrmaRuntimeHelperTest, + copied_send_coalesces_host_fragments_and_retains_aligned_payload) { g_copy_test_releases = 0; butil::IOBuf buffer; for (const char* text : {"first", "second"}) { @@ -197,22 +208,27 @@ TEST(UrmaRuntimeHelperTest, copied_send_coalesces_host_fragments_and_retains_ali } ASSERT_EQ(2u, buffer.backing_block_num()); urma_sge_t sge{}; - ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + ASSERT_TRUE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); EXPECT_EQ(2, g_copy_test_releases); EXPECT_EQ("firstsecond", buffer.to_string()); ASSERT_EQ(1u, buffer.backing_block_num()); EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); - EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), sge.addr); + EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), + sge.addr); EXPECT_EQ(buffer.size(), sge.len); EXPECT_EQ(&g_copy_test_segment, sge.tseg); EXPECT_EQ(nullptr, sge.user_tseg); - EXPECT_EQ("firstsecond", std::string(reinterpret_cast(sge.addr), sge.len)); + EXPECT_EQ("firstsecond", + std::string(reinterpret_cast(sge.addr), sge.len)); buffer.clear(); - const std::string large(urma::GetUrmaRecvBlockSize() - - (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), 'x'); + const std::string large( + urma::GetUrmaRecvBlockSize() - (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), + 'x'); buffer.append(large); - ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + ASSERT_TRUE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); EXPECT_EQ(large, buffer.to_string()); EXPECT_EQ(large.size(), sge.len); @@ -222,13 +238,15 @@ TEST(UrmaRuntimeHelperTest, copied_send_failure_preserves_source_payload) { butil::IOBuf buffer; buffer.append("unchanged"); urma_sge_t sge{}; - EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); + EXPECT_FALSE( + urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); EXPECT_EQ(ERDMAMEM, errno); EXPECT_EQ("unchanged", buffer.to_string()); const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); buffer.clear(); buffer.append(large); - EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_FALSE( + urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); EXPECT_EQ(EMSGSIZE, errno); EXPECT_EQ(large, buffer.to_string()); } @@ -255,7 +273,8 @@ class UrmaRuntimeMockTest : public ::testing::Test { EXPECT_EQ(URMA_SUCCESS, urma_uninit()); _owns_urma_init = false; } - GTEST_SKIP() << "UrmaRuntimeMockTest requires the URMA link-time mock"; + GTEST_SKIP() + << "UrmaRuntimeMockTest requires the URMA link-time mock"; } } @@ -387,9 +406,11 @@ TEST_F(UrmaRecoveryTest, recovery_mode_and_metrics) { EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); EXPECT_NE(std::string::npos, - bvar::Variable::describe_exposed("urma_recovery_status").find("tcp_only")); + bvar::Variable::describe_exposed("urma_recovery_status") + .find("tcp_only")); EXPECT_FALSE(bvar::Variable::describe_exposed( - "urma_recovery_tcp_fallback_admissions").empty()); + "urma_recovery_tcp_fallback_admissions") + .empty()); } TEST_F(UrmaRecoveryTest, rejects_mismatched_path_before_importing_resources) { @@ -414,19 +435,25 @@ TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { urma::UrmaRecoveryTestPeer::Establish(&ep); const auto before = urma::GetUrmaRecoveryStatus(); EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); - const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR ? - urma::UrmaFailureReason::RETRY_EXHAUSTED : - status == URMA_CR_RNR_RETRY_CNT_EXC_ERR ? urma::UrmaFailureReason::RNR : - status == URMA_CR_LOC_ACCESS_ERR ? urma::UrmaFailureReason::ACCESS_ERROR : - urma::UrmaFailureReason::TRANSPORT_ERROR; + const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR + ? urma::UrmaFailureReason::RETRY_EXHAUSTED + : status == URMA_CR_RNR_RETRY_CNT_EXC_ERR + ? urma::UrmaFailureReason::RNR + : status == URMA_CR_LOC_ACCESS_ERR + ? urma::UrmaFailureReason::ACCESS_ERROR + : urma::UrmaFailureReason::TRANSPORT_ERROR; EXPECT_EQ(before.failures[static_cast(reason)] + 1, - urma::GetUrmaRecoveryStatus().failures[static_cast(reason)]); + urma::GetUrmaRecoveryStatus() + .failures[static_cast(reason)]); EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, - urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); - EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)], - urma::GetUrmaRecoveryStatus().failures[ - static_cast(urma::UrmaFailureReason::CQ_POLL)]); + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection( + "127.0.0.1:8123")); + urma::UrmaRecoveryTestPeer::Report(&ep, + urma::UrmaFailureReason::CQ_POLL); + EXPECT_EQ(before.failures[static_cast( + urma::UrmaFailureReason::CQ_POLL)], + urma::GetUrmaRecoveryStatus().failures[static_cast( + urma::UrmaFailureReason::CQ_POLL)]); socket->SetFailed(); } } @@ -437,12 +464,15 @@ TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { urma::UrmaEndpoint ep(socket.get()); urma::UrmaRecoveryTestPeer::Establish(&ep); EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); - EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); - EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ(-1, + urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); + EXPECT_TRUE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); - EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_TRUE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); socket->SetFailed(); } @@ -452,7 +482,8 @@ TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { urma::UrmaEndpoint ep(socket.get()); urma::UrmaRecoveryTestPeer::Establish(&ep); EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); - EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_FALSE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); socket->SetFailed(); } @@ -464,15 +495,20 @@ TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { const auto before = urma::GetUrmaRecoveryStatus(); urma::UrmaRecoveryTestPeer::PollFailure(&ep); EXPECT_TRUE(socket->Failed()); - EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); - EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + 1, - urma::GetUrmaRecoveryStatus().failures[ - static_cast(urma::UrmaFailureReason::CQ_POLL)]); + EXPECT_FALSE( + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ( + before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + + 1, + urma::GetUrmaRecoveryStatus() + .failures[static_cast(urma::UrmaFailureReason::CQ_POLL)]); } class UrmaRecoveryEcho : public ::test::EchoService { - void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, - ::test::EchoResponse* response, google::protobuf::Closure* done) override { + void Echo(google::protobuf::RpcController*, + const ::test::EchoRequest* request, + ::test::EchoResponse* response, + google::protobuf::Closure* done) override { response->set_message(request->message()); done->Run(); } @@ -512,7 +548,7 @@ TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { urma::UrmaEndpoint failed_ep(failed_socket.get()); urma::UrmaRecoveryTestPeer::Establish(&failed_ep); EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( - &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); + &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); } failed_socket->SetFailed(); const auto before = urma::GetUrmaRecoveryStatus(); @@ -534,10 +570,12 @@ TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { so.socket_mode = SOCKET_MODE_URMA; ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); for (int version : {2, 3}) { - butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr)); + butil::fd_guard fd( + butil::tcp_connect(server.listen_address(), nullptr)); ASSERT_GE(fd, 0); timeval timeout{2, 0}; - ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); + ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout))); urma::v2_wire::HelloMessage hello; hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; @@ -554,9 +592,12 @@ TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; memcpy(packet, "URMA", 4); hello.Serialize(packet + 4); - ASSERT_EQ(sizeof(packet), static_cast(write(fd, packet, sizeof(packet)))); + ASSERT_EQ(sizeof(packet), + static_cast(write(fd, packet, sizeof(packet)))); uint8_t reply[sizeof(packet)]; - ASSERT_EQ(sizeof(reply), static_cast(recv(fd, reply, sizeof(reply), MSG_WAITALL))); + ASSERT_EQ(sizeof(reply), + static_cast( + recv(fd, reply, sizeof(reply), MSG_WAITALL))); EXPECT_EQ(0, memcmp(reply, "URMA", 4)); urma::v2_wire::HelloMessage rejected; rejected.Deserialize(reply + 4); @@ -580,7 +621,8 @@ TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { std::string packet("URM3", 4); packet.append(reinterpret_cast(&size), 4); packet.append(body); - ASSERT_EQ(packet.size(), static_cast(write(fd, packet.data(), packet.size()))); + ASSERT_EQ(packet.size(), static_cast(write( + fd, packet.data(), packet.size()))); char header[8]; ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); EXPECT_EQ(0, memcmp(header, "URM3", 4)); @@ -588,13 +630,16 @@ TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { size = butil::NetToHost32(size); ASSERT_LE(size, 4096u); std::string reply(size, '\0'); - ASSERT_EQ(size, static_cast(recv(fd, &reply[0], size, MSG_WAITALL))); + ASSERT_EQ(size, static_cast( + recv(fd, &reply[0], size, MSG_WAITALL))); urma::UrmaHello rejected; ASSERT_TRUE(rejected.ParseFromString(reply)); EXPECT_EQ(0u, rejected.jetty_id()); } - const std::string request = "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; - ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), request.size()))); + const std::string request = + "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), + request.size()))); char response[4096]; const ssize_t nr = recv(fd, response, sizeof(response), 0); ASSERT_GT(nr, 0); diff --git a/test/brpc_urma_transport_unittest.cpp b/test/brpc_urma_transport_unittest.cpp index f7ca8e87b8..e1193fde0f 100644 --- a/test/brpc_urma_transport_unittest.cpp +++ b/test/brpc_urma_transport_unittest.cpp @@ -42,8 +42,9 @@ DECLARE_int32(urma_client_handshake_version); namespace { class PathEchoService : public test::EchoService { - void Echo(google::protobuf::RpcController* base, const test::EchoRequest* request, - test::EchoResponse* response, google::protobuf::Closure* done) override { + void Echo(google::protobuf::RpcController* base, + const test::EchoRequest* request, test::EchoResponse* response, + google::protobuf::Closure* done) override { brpc::ClosureGuard guard(done); auto* cntl = static_cast(base); response->set_message(request->message()); @@ -62,7 +63,7 @@ class UrmaTransportTest : public ::testing::TestWithParam { int count = 0; auto** devices = urma_get_device_list(&count); _mock = devices && count > 0 && - strcmp(devices[0]->name, "mock_urma_device") == 0; + strcmp(devices[0]->name, "mock_urma_device") == 0; urma_free_device_list(devices); if (status == URMA_SUCCESS) { urma_uninit(); @@ -123,7 +124,8 @@ TEST_P(UrmaTransportTest, exchanges_zero_copy_payloads_and_receive_credits) { test::EchoResponse response; request.set_message(payload); cntl.request_attachment().append(payload); - test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); EXPECT_EQ(payload, response.message()); EXPECT_EQ(payload, cntl.response_attachment().to_string()); @@ -134,7 +136,8 @@ TEST_P(UrmaTransportTest, exchanges_zero_copy_payloads_and_receive_credits) { server.Join(); } -TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_back) { +TEST_P(UrmaTransportTest, + switches_new_connections_from_auto_to_tcp_only_and_back) { PathEchoService service; brpc::Server server; ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); @@ -161,13 +164,15 @@ TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_bac test::EchoRequest request; test::EchoResponse response; request.set_message("switch"); - test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, + nullptr); ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); EXPECT_EQ("switch", response.message()); } const auto after = brpc::urma::GetUrmaRecoveryStatus(); if (mode == brpc::urma::UrmaRecoveryMode::TCP_ONLY) { - EXPECT_EQ(before.negotiated_connections, after.negotiated_connections); + EXPECT_EQ(before.negotiated_connections, + after.negotiated_connections); EXPECT_EQ(before.tcp_only_admissions + requests_per_mode, after.tcp_only_admissions); } else { @@ -180,7 +185,8 @@ TEST_P(UrmaTransportTest, switches_new_connections_from_auto_to_tcp_only_and_bac server.Join(); } -INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaTransportTest, ::testing::Values(2, 3)); +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaTransportTest, + ::testing::Values(2, 3)); } // namespace #endif // BRPC_WITH_URMA From a00e29374ca9d56d72e7206c14d89285f4a73b57 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:26:07 -0800 Subject: [PATCH 25/29] Install gperftools for URMA mock CI --- .github/workflows/ci-urma.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-urma.yml b/.github/workflows/ci-urma.yml index 0ca65de7dc..4b8fe52bee 100644 --- a/.github/workflows/ci-urma.yml +++ b/.github/workflows/ci-urma.yml @@ -54,7 +54,7 @@ jobs: sudo apt-get update sudo apt-get install -y ninja-build libssl-dev libgflags-dev \ libprotobuf-dev libprotoc-dev protobuf-compiler libleveldb-dev \ - libgtest-dev libgmock-dev + libgtest-dev libgmock-dev libgoogle-perftools-dev - name: Configure run: | cmake -S . -B build/urma-ci -G Ninja \ From 351935af1def7477b121b2335e010448bf6c65de Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:15:49 -0800 Subject: [PATCH 26/29] Publish mock URMA SEND completions after payload delivery --- .github/workflows/ci-urma.yml | 3 +- src/brpc/urma/mock_urma.cpp | 23 ++- test/brpc_urma_mock_completion_unittest.cpp | 162 ++++++++++++++++++++ 3 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 test/brpc_urma_mock_completion_unittest.cpp diff --git a/.github/workflows/ci-urma.yml b/.github/workflows/ci-urma.yml index 4b8fe52bee..174ddb9a2c 100644 --- a/.github/workflows/ci-urma.yml +++ b/.github/workflows/ci-urma.yml @@ -66,7 +66,8 @@ jobs: run: | targets=(brpc-static brpc_urma_unittest brpc_urma_recovery_unittest brpc_urma_runtime_unittest brpc_urma_copy_unittest - brpc_urma_transport_unittest brpc_urma_remote_recovery_unittest) + brpc_urma_transport_unittest brpc_urma_remote_recovery_unittest + brpc_urma_mock_completion_unittest) if test -f test/brpc_urma_qos_unittest.cpp; then targets+=(brpc_urma_qos_unittest brpc_urma_qos_churn_unittest) fi diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index 4263b4d640..51e1e4d089 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -572,14 +572,16 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, return URMA_EINVAL; } - urma_cr_t send_cr{}; - send_cr.status = URMA_CR_SUCCESS; - send_cr.user_ctx = current->user_ctx; - send_cr.flag.bs.s_r = 0; - if (current->flag.bs.complete_enable) { - PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, - local_state, send_cr); - } + auto complete_send = [&] { + if (current->flag.bs.complete_enable) { + urma_cr_t send_cr{}; + send_cr.status = URMA_CR_SUCCESS; + send_cr.user_ctx = current->user_ctx; + send_cr.flag.bs.s_r = 0; + PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, + local_state, send_cr); + } + }; PendingRecv recv{}; urma_jfc_t* remote_jfc = nullptr; @@ -587,6 +589,7 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, std::unique_lock lock(g_rw_mutex); auto remote_it = jetty_id_map.find(current->tjetty->id.id); if (remote_it == jetty_id_map.end()) { + complete_send(); continue; } urma_jfr_t* remote_jfr = @@ -595,6 +598,7 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, auto jfc_it = jfr_jfc_map.find(remote_jfr); if (recv_it == jfr_recv_map.end() || recv_it->second.empty() || jfc_it == jfr_jfc_map.end()) { + complete_send(); continue; } recv = recv_it->second.front(); @@ -633,6 +637,9 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, recv_cr.imm_data = current->send.imm_data; PushCompletion(remote_jfc, remote_state, recv_cr); } + // A SEND completion lets the consumer release its source buffers. + // Finish copying the payload before publishing that completion. + complete_send(); } if (bad_wr) { *bad_wr = nullptr; diff --git a/test/brpc_urma_mock_completion_unittest.cpp b/test/brpc_urma_mock_completion_unittest.cpp new file mode 100644 index 0000000000..5a6173fb82 --- /dev/null +++ b/test/brpc_urma_mock_completion_unittest.cpp @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "urma_api.h" + +namespace { + +class UrmaMockCompletionTest : public testing::Test { + protected: + void SetUp() override { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + _owns_init = status == URMA_SUCCESS; + int count = 0; + auto** devices = urma_get_device_list(&count); + const bool mock = + devices && count > 0 && + std::strcmp(devices[0]->name, "mock_urma_device") == 0; + if (mock) { + _context = urma_create_context(devices[0], 0); + } + urma_free_device_list(devices); + if (!mock) { + GTEST_SKIP() << "Completion ordering test uses the link-time mock"; + } + ASSERT_NE(nullptr, _context); + urma_jfc_cfg_t cq{}; + cq.depth = 8; + _send_cq = urma_create_jfc(_context, &cq); + _recv_cq = urma_create_jfc(_context, &cq); + ASSERT_NE(nullptr, _send_cq); + ASSERT_NE(nullptr, _recv_cq); + urma_jfr_cfg_t rq{}; + rq.depth = 8; + rq.jfc = _recv_cq; + _recv_queue = urma_create_jfr(_context, &rq); + ASSERT_NE(nullptr, _recv_queue); + urma_jetty_cfg_t cfg{}; + cfg.jfs_cfg.jfc = _send_cq; + cfg.shared.jfr = _recv_queue; + _sender = urma_create_jetty(_context, &cfg); + _receiver = urma_create_jetty(_context, &cfg); + ASSERT_NE(nullptr, _sender); + ASSERT_NE(nullptr, _receiver); + urma_rjetty_t remote{}; + remote.jetty_id = _receiver->jetty_id; + urma_token_t token{}; + _target = urma_import_jetty(_context, &remote, &token); + ASSERT_NE(nullptr, _target); + } + + void TearDown() override { + if (_target) urma_unimport_jetty(_target); + if (_receiver) urma_delete_jetty(_receiver); + if (_sender) urma_delete_jetty(_sender); + if (_recv_queue) urma_delete_jfr(_recv_queue); + if (_recv_cq) urma_delete_jfc(_recv_cq); + if (_send_cq) urma_delete_jfc(_send_cq); + if (_context) urma_delete_context(_context); + if (_owns_init) urma_uninit(); + } + + bool _owns_init = false; + urma_context_t* _context = nullptr; + urma_jfc_t* _send_cq = nullptr; + urma_jfc_t* _recv_cq = nullptr; + urma_jfr_t* _recv_queue = nullptr; + urma_jetty_t* _sender = nullptr; + urma_jetty_t* _receiver = nullptr; + urma_target_jetty_t* _target = nullptr; +}; + +TEST_F(UrmaMockCompletionTest, send_completion_follows_payload_delivery) { + // Keep the copy large enough for a concurrent CQ consumer to run while + // the mock is still using the source buffer. + const std::string payload(16 * 1024 * 1024, 'p'); + std::string received(payload.size(), '\0'); + urma_sge_t recv_sge{}; + recv_sge.addr = reinterpret_cast(&received[0]); + recv_sge.len = received.size(); + urma_jfr_wr_t recv_wr{}; + recv_wr.src.sge = &recv_sge; + recv_wr.src.num_sge = 1; + recv_wr.user_ctx = 2; + urma_jfr_wr_t* bad_recv = nullptr; + ASSERT_EQ(URMA_SUCCESS, urma_post_jfr_wr(_recv_queue, &recv_wr, &bad_recv)); + + urma_sge_t send_sge{}; + send_sge.addr = reinterpret_cast(payload.data()); + send_sge.len = payload.size(); + urma_jfs_wr_t send_wr{}; + send_wr.opcode = URMA_OPC_SEND; + send_wr.flag.bs.complete_enable = 1; + send_wr.tjetty = _target; + send_wr.send.src.sge = &send_sge; + send_wr.send.src.num_sge = 1; + send_wr.user_ctx = 1; + std::atomic started{false}; + std::atomic finished{false}; + urma_status_t posted = URMA_EINVAL; + std::thread writer([&] { + while (!started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + urma_jfs_wr_t* bad_send = nullptr; + posted = urma_post_jetty_send_wr(_sender, &send_wr, &bad_send); + finished.store(true, std::memory_order_release); + }); + started.store(true, std::memory_order_release); + urma_cr_t send_cr{}; + int send_count = 0; + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (send_count == 0 && std::chrono::steady_clock::now() < deadline) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + if (finished.load(std::memory_order_acquire) && send_count == 0) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + break; + } + } + urma_cr_t recv_cr{}; + const int recv_count = urma_poll_jfc(_recv_cq, 1, &recv_cr); + // Join before assertions so a failed check still releases the worker. + writer.join(); + ASSERT_EQ(URMA_SUCCESS, posted); + ASSERT_EQ(1, send_count); + EXPECT_EQ(URMA_CR_SUCCESS, send_cr.status); + EXPECT_EQ(1u, send_cr.user_ctx); + ASSERT_EQ(1, recv_count) + << "Mock SEND completed before delivering its source payload"; + EXPECT_EQ(URMA_CR_SUCCESS, recv_cr.status); + EXPECT_EQ(2u, recv_cr.user_ctx); + EXPECT_EQ(payload.size(), recv_cr.completion_len); + EXPECT_EQ(payload, received); +} + +} // namespace +#endif // BRPC_WITH_URMA From 040fb3334f89da2fe17fe354f13125a48cc9eb09 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Thu, 5 Nov 2026 04:15:49 -0800 Subject: [PATCH 27/29] Publish mock URMA SEND completions after payload delivery (cherry picked from commit 351935af1def7477b121b2335e010448bf6c65de) --- .github/workflows/ci-urma.yml | 3 +- src/brpc/urma/mock_urma.cpp | 23 ++- test/brpc_urma_mock_completion_unittest.cpp | 162 ++++++++++++++++++++ 3 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 test/brpc_urma_mock_completion_unittest.cpp diff --git a/.github/workflows/ci-urma.yml b/.github/workflows/ci-urma.yml index 4b8fe52bee..174ddb9a2c 100644 --- a/.github/workflows/ci-urma.yml +++ b/.github/workflows/ci-urma.yml @@ -66,7 +66,8 @@ jobs: run: | targets=(brpc-static brpc_urma_unittest brpc_urma_recovery_unittest brpc_urma_runtime_unittest brpc_urma_copy_unittest - brpc_urma_transport_unittest brpc_urma_remote_recovery_unittest) + brpc_urma_transport_unittest brpc_urma_remote_recovery_unittest + brpc_urma_mock_completion_unittest) if test -f test/brpc_urma_qos_unittest.cpp; then targets+=(brpc_urma_qos_unittest brpc_urma_qos_churn_unittest) fi diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index f6b89db1ec..28ae74423c 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -566,14 +566,16 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, return URMA_EINVAL; } - urma_cr_t send_cr{}; - send_cr.status = URMA_CR_SUCCESS; - send_cr.user_ctx = current->user_ctx; - send_cr.flag.bs.s_r = 0; - if (current->flag.bs.complete_enable) { - PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, - local_state, send_cr); - } + auto complete_send = [&] { + if (current->flag.bs.complete_enable) { + urma_cr_t send_cr{}; + send_cr.status = URMA_CR_SUCCESS; + send_cr.user_ctx = current->user_ctx; + send_cr.flag.bs.s_r = 0; + PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, + local_state, send_cr); + } + }; PendingRecv recv{}; urma_jfc_t* remote_jfc = nullptr; @@ -581,6 +583,7 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, std::unique_lock lock(g_rw_mutex); auto remote_it = jetty_id_map.find(current->tjetty->id.id); if (remote_it == jetty_id_map.end()) { + complete_send(); continue; } urma_jfr_t* remote_jfr = @@ -589,6 +592,7 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, auto jfc_it = jfr_jfc_map.find(remote_jfr); if (recv_it == jfr_recv_map.end() || recv_it->second.empty() || jfc_it == jfr_jfc_map.end()) { + complete_send(); continue; } recv = recv_it->second.front(); @@ -627,6 +631,9 @@ urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, recv_cr.imm_data = current->send.imm_data; PushCompletion(remote_jfc, remote_state, recv_cr); } + // A SEND completion lets the consumer release its source buffers. + // Finish copying the payload before publishing that completion. + complete_send(); } if (bad_wr) { *bad_wr = nullptr; diff --git a/test/brpc_urma_mock_completion_unittest.cpp b/test/brpc_urma_mock_completion_unittest.cpp new file mode 100644 index 0000000000..5a6173fb82 --- /dev/null +++ b/test/brpc_urma_mock_completion_unittest.cpp @@ -0,0 +1,162 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "urma_api.h" + +namespace { + +class UrmaMockCompletionTest : public testing::Test { + protected: + void SetUp() override { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + ASSERT_TRUE(status == URMA_SUCCESS || status == URMA_EEXIST); + _owns_init = status == URMA_SUCCESS; + int count = 0; + auto** devices = urma_get_device_list(&count); + const bool mock = + devices && count > 0 && + std::strcmp(devices[0]->name, "mock_urma_device") == 0; + if (mock) { + _context = urma_create_context(devices[0], 0); + } + urma_free_device_list(devices); + if (!mock) { + GTEST_SKIP() << "Completion ordering test uses the link-time mock"; + } + ASSERT_NE(nullptr, _context); + urma_jfc_cfg_t cq{}; + cq.depth = 8; + _send_cq = urma_create_jfc(_context, &cq); + _recv_cq = urma_create_jfc(_context, &cq); + ASSERT_NE(nullptr, _send_cq); + ASSERT_NE(nullptr, _recv_cq); + urma_jfr_cfg_t rq{}; + rq.depth = 8; + rq.jfc = _recv_cq; + _recv_queue = urma_create_jfr(_context, &rq); + ASSERT_NE(nullptr, _recv_queue); + urma_jetty_cfg_t cfg{}; + cfg.jfs_cfg.jfc = _send_cq; + cfg.shared.jfr = _recv_queue; + _sender = urma_create_jetty(_context, &cfg); + _receiver = urma_create_jetty(_context, &cfg); + ASSERT_NE(nullptr, _sender); + ASSERT_NE(nullptr, _receiver); + urma_rjetty_t remote{}; + remote.jetty_id = _receiver->jetty_id; + urma_token_t token{}; + _target = urma_import_jetty(_context, &remote, &token); + ASSERT_NE(nullptr, _target); + } + + void TearDown() override { + if (_target) urma_unimport_jetty(_target); + if (_receiver) urma_delete_jetty(_receiver); + if (_sender) urma_delete_jetty(_sender); + if (_recv_queue) urma_delete_jfr(_recv_queue); + if (_recv_cq) urma_delete_jfc(_recv_cq); + if (_send_cq) urma_delete_jfc(_send_cq); + if (_context) urma_delete_context(_context); + if (_owns_init) urma_uninit(); + } + + bool _owns_init = false; + urma_context_t* _context = nullptr; + urma_jfc_t* _send_cq = nullptr; + urma_jfc_t* _recv_cq = nullptr; + urma_jfr_t* _recv_queue = nullptr; + urma_jetty_t* _sender = nullptr; + urma_jetty_t* _receiver = nullptr; + urma_target_jetty_t* _target = nullptr; +}; + +TEST_F(UrmaMockCompletionTest, send_completion_follows_payload_delivery) { + // Keep the copy large enough for a concurrent CQ consumer to run while + // the mock is still using the source buffer. + const std::string payload(16 * 1024 * 1024, 'p'); + std::string received(payload.size(), '\0'); + urma_sge_t recv_sge{}; + recv_sge.addr = reinterpret_cast(&received[0]); + recv_sge.len = received.size(); + urma_jfr_wr_t recv_wr{}; + recv_wr.src.sge = &recv_sge; + recv_wr.src.num_sge = 1; + recv_wr.user_ctx = 2; + urma_jfr_wr_t* bad_recv = nullptr; + ASSERT_EQ(URMA_SUCCESS, urma_post_jfr_wr(_recv_queue, &recv_wr, &bad_recv)); + + urma_sge_t send_sge{}; + send_sge.addr = reinterpret_cast(payload.data()); + send_sge.len = payload.size(); + urma_jfs_wr_t send_wr{}; + send_wr.opcode = URMA_OPC_SEND; + send_wr.flag.bs.complete_enable = 1; + send_wr.tjetty = _target; + send_wr.send.src.sge = &send_sge; + send_wr.send.src.num_sge = 1; + send_wr.user_ctx = 1; + std::atomic started{false}; + std::atomic finished{false}; + urma_status_t posted = URMA_EINVAL; + std::thread writer([&] { + while (!started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + urma_jfs_wr_t* bad_send = nullptr; + posted = urma_post_jetty_send_wr(_sender, &send_wr, &bad_send); + finished.store(true, std::memory_order_release); + }); + started.store(true, std::memory_order_release); + urma_cr_t send_cr{}; + int send_count = 0; + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (send_count == 0 && std::chrono::steady_clock::now() < deadline) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + if (finished.load(std::memory_order_acquire) && send_count == 0) { + send_count = urma_poll_jfc(_send_cq, 1, &send_cr); + break; + } + } + urma_cr_t recv_cr{}; + const int recv_count = urma_poll_jfc(_recv_cq, 1, &recv_cr); + // Join before assertions so a failed check still releases the worker. + writer.join(); + ASSERT_EQ(URMA_SUCCESS, posted); + ASSERT_EQ(1, send_count); + EXPECT_EQ(URMA_CR_SUCCESS, send_cr.status); + EXPECT_EQ(1u, send_cr.user_ctx); + ASSERT_EQ(1, recv_count) + << "Mock SEND completed before delivering its source payload"; + EXPECT_EQ(URMA_CR_SUCCESS, recv_cr.status); + EXPECT_EQ(2u, recv_cr.user_ctx); + EXPECT_EQ(payload.size(), recv_cr.completion_len); + EXPECT_EQ(payload, received); +} + +} // namespace +#endif // BRPC_WITH_URMA From 004d3aea3758f256ed6ce28603f351cf5a7a01d4 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 16 Nov 2026 01:55:43 -0800 Subject: [PATCH 28/29] Document URMA QoS mappings and Jetty costs --- docs/cn/urma.md | 19 ++++++++++++------- docs/en/urma.md | 34 +++++++++++++++++++++------------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/docs/cn/urma.md b/docs/cn/urma.md index d90447e5a6..0c491c10cc 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -259,7 +259,9 @@ URMA 恢复及 64 KiB 附件回显。正常短连接关闭也在同一测试中 ## 业务 SL 与优先级 -`ChannelOptions.urma_qos` 控制请求发送方向,`ServerOptions.urma_qos` 控制响应发送方向: +同一进程可以同时运行对延迟敏感的 RPC 和批量传输。两类业务可以分别设置发送优先级, +未显式配置的业务继承进程默认值。`ChannelOptions.urma_qos` 控制请求发送方向, +`ServerOptions.urma_qos` 控制响应发送方向。 ```cpp brpc::ChannelOptions client_options; @@ -280,14 +282,17 @@ rtp 使用 priority 0。 | `service_level` | -1 | 0–255 | 选择当前路径下首个映射到该 SL 的 priority | | `priority` | -1 | 0–15 | 指定当前路径支持的 priority | -两项均为 -1 时使用设备默认 priority;同时指定时须匹配同一设备映射。 -不支持或冲突时,Channel 初始化、Server 启动报错,并列出当前路径支持的 priority:SL。 -例如 RTP priority 0→SL 2 时配置 `service_level=2`,Jetty 使用 priority 0。 +两项均为 -1 时使用进程默认 priority。同时指定时,两项须匹配同一设备映射。 +每个设备在所选路径下最多提供 16 个 priority/SL 映射,实际可用项取决于设备能力。 +显式配置不受支持或发生冲突时,Channel 初始化或 Server 启动失败,日志会列出当前路径支持的映射。 平台配置设备映射与调度策略,brpc 使用进程初始化时的映射快照。 -不同配置使用独立连接,相同配置共享连接。配置随服务发现、连接池、短连接和 -服务端 accept 传递,Jetty 重建时重新应用。预建池服务于默认 priority, -其他 priority 按需创建。v2/v3 握手使用所选路径。 +客户端连接签名包含原始 QoS 设置。即使 RTP 的 priority 0 是 SL 2 的首个匹配项, +只填 `service_level=2` 和同时填 `priority=0` 的两个 Channel 也会分别建连。 +配置随服务发现、连接池、短连接和服务端 accept 传递,Jetty 重建时重新应用。 + +活跃的 URMA 连接在每端各占用一个 Jetty。匹配默认 priority 和队列大小的连接可以使用 +预建池中的 Jetty,其他连接按需创建 Jetty。v2/v3 握手使用所选路径。 ### 功能验证 diff --git a/docs/en/urma.md b/docs/en/urma.md index 7c1c999686..3252e9db1b 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -282,8 +282,10 @@ failure logs report direction, send slot and registered segment range. ## Business SL and priority -Set `ChannelOptions.urma_qos` for outgoing requests and -`ServerOptions.urma_qos` for outgoing responses: +A process can serve latency-sensitive RPCs alongside bulk transfers. Configure +their outgoing priorities independently and leave other traffic on the process +default. Set `ChannelOptions.urma_qos` for outgoing requests and +`ServerOptions.urma_qos` for outgoing responses. ```cpp brpc::ChannelOptions client_options; @@ -305,17 +307,23 @@ priority 6 for auto/CTP and priority 0 for RTP. | `service_level` | -1 | 0–255 | Select the first priority mapping to this SL on the selected path | | `priority` | -1 | 0–15 | Select a priority supported by the selected path | -Both fields at -1 select the device default. When both are explicit, they must -match the same device mapping. Channel initialization and Server startup reject -unsupported or conflicting settings and report supported priority:SL pairs. -For example, RTP priority 0 mapping to SL 2 resolves `service_level=2` to priority 0. -The platform configures mappings and scheduling; brpc uses the initialization snapshot. - -Different configurations use separate connections; equal configurations share -connections. Naming services, pooled and short connections, accepted sockets and -rebuilt Jettys inherit the configuration. The prepared pool serves the default -priority; other priorities allocate Jettys on demand. Both v2/v3 handshakes carry -the selected path. +Both fields at -1 select the process default. When both are explicit, they must +match the same device mapping. Each device exposes up to 16 priority/SL mappings +on the selected path, depending on its capabilities. Channel initialization and +Server startup reject unsupported or conflicting settings and log the available +mappings. The platform configures mappings and scheduling. brpc uses the +initialization snapshot. + +The client connection signature includes the configured QoS values. If RTP +priority 0 is the first mapping for SL 2, a channel that sets only +`service_level=2` uses a separate connection from one that also sets +`priority=0`. Naming services, pooled and short connections, accepted sockets +and rebuilt Jettys inherit the configuration. + +Each active URMA connection uses one Jetty at each end. Connections with the +default priority and a compatible queue size can use a Jetty from the prepared +pool. Other connections allocate Jettys on demand. Both v2 and v3 use the +selected path. ### Functional validation From cf88d72886a35716dc4b3fbb9e2282c530747846 Mon Sep 17 00:00:00 2001 From: Spicy-cream <2667314914@qq.com> Date: Mon, 16 Nov 2026 22:53:59 -0800 Subject: [PATCH 29/29] Fix URMA teardown and SEND retry handling --- src/brpc/urma/mock_urma.cpp | 137 +++++----- src/brpc/urma/urma_endpoint.cpp | 261 +++++++++++--------- src/brpc/urma/urma_endpoint.h | 8 +- src/brpc/urma/urma_helper.cpp | 4 + src/brpc/urma_transport.cpp | 5 +- test/brpc_urma_mock_completion_unittest.cpp | 98 ++++++++ test/brpc_urma_runtime_unittest.cpp | 15 +- 7 files changed, 327 insertions(+), 201 deletions(-) diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index 28ae74423c..49c705dee4 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -543,97 +543,82 @@ urma_status_t urma_modify_jetty(urma_jetty_t *jetty, urma_jetty_attr_t *attr) { urma_status_t urma_post_jetty_send_wr(urma_jetty_t *jetty, urma_jfs_wr_t *wr, urma_jfs_wr_t **bad_wr) { - std::shared_lock read_lock(g_rw_mutex); - auto local_it = jetty_map.find(jetty); - auto local_jfc_it = - jetty ? jfc_state_map.find(jetty->jetty_cfg.jfs_cfg.jfc) - : jfc_state_map.end(); - if (!jetty || !wr || local_it == jetty_map.end() || - local_jfc_it == jfc_state_map.end()) { + // Protect the target, pending receive buffer and both completion queues + // from teardown until delivery and notification are complete. + std::unique_lock lock(g_rw_mutex); + if (!jetty || !wr || jetty_map.find(jetty) == jetty_map.end()) { + if (bad_wr) { + *bad_wr = wr; + } + return URMA_EINVAL; + } + urma_jfc_t* local_jfc = jetty->jetty_cfg.jfs_cfg.jfc; + const auto local_state = jfc_state_map.find(local_jfc); + if (local_state == jfc_state_map.end()) { if (bad_wr) { *bad_wr = wr; } return URMA_EINVAL; } - JfcState* local_state = local_jfc_it->second; - read_lock.unlock(); for (urma_jfs_wr_t* current = wr; current; current = current->next) { - if (!current->tjetty) { - if (bad_wr) { - *bad_wr = current; - } + if (bad_wr) { + *bad_wr = current; + } + if (target_jetty_map.find(current->tjetty) == target_jetty_map.end()) { return URMA_EINVAL; } - - auto complete_send = [&] { - if (current->flag.bs.complete_enable) { - urma_cr_t send_cr{}; - send_cr.status = URMA_CR_SUCCESS; - send_cr.user_ctx = current->user_ctx; - send_cr.flag.bs.s_r = 0; - PushCompletion(jetty->jetty_cfg.jfs_cfg.jfc, - local_state, send_cr); - } - }; - - PendingRecv recv{}; - urma_jfc_t* remote_jfc = nullptr; - { - std::unique_lock lock(g_rw_mutex); - auto remote_it = jetty_id_map.find(current->tjetty->id.id); - if (remote_it == jetty_id_map.end()) { - complete_send(); - continue; - } - urma_jfr_t* remote_jfr = - remote_it->second->jetty_cfg.shared.jfr; - auto recv_it = jfr_recv_map.find(remote_jfr); - auto jfc_it = jfr_jfc_map.find(remote_jfr); - if (recv_it == jfr_recv_map.end() || recv_it->second.empty() || - jfc_it == jfr_jfc_map.end()) { - complete_send(); - continue; - } - recv = recv_it->second.front(); - recv_it->second.pop_front(); - remote_jfc = jfc_it->second; + const auto remote = jetty_id_map.find(current->tjetty->id.id); + if (remote == jetty_id_map.end()) { + return URMA_FAIL; + } + urma_jfr_t* jfr = remote->second->jetty_cfg.shared.jfr; + const auto pending = jfr_recv_map.find(jfr); + const auto cq = jfr_jfc_map.find(jfr); + if (pending == jfr_recv_map.end() || cq == jfr_jfc_map.end()) { + return URMA_FAIL; + } + const auto remote_state = jfc_state_map.find(cq->second); + if (remote_state == jfc_state_map.end()) { + return URMA_FAIL; + } + if (pending->second.empty()) { + return URMA_EAGAIN; } + const PendingRecv recv = pending->second.front(); + const auto& src = current->send.src; + uint64_t length = 0; + for (uint32_t i = 0; i < src.num_sge; ++i) { + length += src.sge[i].len; + } + if (length > recv.len) { + return URMA_EINVAL; + } + pending->second.pop_front(); uint32_t copied = 0; - for (uint32_t i = 0; - i < current->send.src.num_sge && copied < recv.len; ++i) { - const urma_sge_t& sge = current->send.src.sge[i]; - const uint32_t n = std::min(sge.len, recv.len - copied); + for (uint32_t i = 0; i < src.num_sge; ++i) { + const auto& sge = src.sge[i]; std::memcpy(reinterpret_cast(recv.addr + copied), - reinterpret_cast(sge.addr), n); - copied += n; - } - - JfcState* remote_state = nullptr; - { - std::shared_lock lock(g_rw_mutex); - auto state_it = jfc_state_map.find(remote_jfc); - if (state_it != jfc_state_map.end()) { - remote_state = state_it->second; - } + reinterpret_cast(sge.addr), sge.len); + copied += sge.len; } - if (remote_state) { - urma_cr_t recv_cr{}; - recv_cr.status = URMA_CR_SUCCESS; - recv_cr.user_ctx = recv.user_ctx; - recv_cr.flag.bs.s_r = 1; - recv_cr.completion_len = copied; - recv_cr.opcode = - current->opcode == URMA_OPC_SEND_IMM - ? URMA_CR_OPC_SEND_WITH_IMM - : URMA_CR_OPC_SEND; - recv_cr.imm_data = current->send.imm_data; - PushCompletion(remote_jfc, remote_state, recv_cr); + urma_cr_t recv_cr{}; + recv_cr.status = URMA_CR_SUCCESS; + recv_cr.user_ctx = recv.user_ctx; + recv_cr.flag.bs.s_r = 1; + recv_cr.completion_len = copied; + recv_cr.opcode = current->opcode == URMA_OPC_SEND_IMM + ? URMA_CR_OPC_SEND_WITH_IMM + : URMA_CR_OPC_SEND; + recv_cr.imm_data = current->send.imm_data; + PushCompletion(cq->second, remote_state->second, recv_cr); + if (current->flag.bs.complete_enable) { + urma_cr_t send_cr{}; + send_cr.status = URMA_CR_SUCCESS; + send_cr.user_ctx = current->user_ctx; + PushCompletion(local_jfc, local_state->second, send_cr); } - // A SEND completion lets the consumer release its source buffers. - // Finish copying the payload before publishing that completion. - complete_send(); } if (bad_wr) { *bad_wr = nullptr; diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index d4119397fb..552ace6692 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -19,6 +19,7 @@ #if BRPC_WITH_URMA +#include #include #include @@ -35,6 +36,7 @@ #include #include "butil/atomicops.h" +#include "butil/fd_guard.h" #include "butil/iobuf.h" #include "butil/logging.h" #include "butil/macros.h" @@ -97,11 +99,12 @@ static int PreparedJettyCount() { } // In event mode each prepared JFCE consumes a file descriptor. Keep room - // for one TCP fd per future URMA connection and for brpc/system internals. + // for a TCP fd and a CQ carrier fd per future URMA connection, plus + // descriptors used by brpc/system internals. static const rlim_t kReservedFdCount = 64; const rlim_t max_prepared = nofile.rlim_cur > kReservedFdCount - ? (nofile.rlim_cur - kReservedFdCount) / 2 + ? (nofile.rlim_cur - kReservedFdCount) / 3 : 0; if (max_prepared >= static_cast(requested)) { return requested; @@ -185,9 +188,6 @@ void UrmaEndpoint::Reset() { _rq_received = 0; _pending_received_bytes.store(0, butil::memory_order_relaxed); _input_processor.Reset(); - _sbuf.clear(); - _rbuf.clear(); - _rbuf_data.clear(); _read_butex->store(0, butil::memory_order_relaxed); } @@ -390,7 +390,21 @@ int UrmaEndpoint::ReadAndParseHelloV3(ParsedHello* out, bool* negotiated) { // Allocate / deallocate per-connection resources. // ============================================================================ +namespace { + +class UrmaCqSocketUser : public SocketUser { +public: + explicit UrmaCqSocketUser(SocketId owner) : owner_id(owner) {} + + void BeforeRecycle(Socket*) override { delete this; } + + const SocketId owner_id; +}; + +} // namespace + int UrmaEndpoint::AllocateResources() { + std::unique_lock lock(_cq_mutex); if (_resource) { return 0; } @@ -474,7 +488,9 @@ int UrmaEndpoint::AllocateResources() { _rbuf.resize(_rq_size); _rbuf_data.resize(_rq_size, nullptr); - // Wrap the JFCE fd in a brpc Socket so PollCq is driven by epoll. + // The provider owns the original JFCE fd. Give the carrier its own + // descriptor so its lifetime does not depend on provider teardown. + butil::fd_guard event_fd; if (!FLAGS_urma_use_polling) { if (!_resource->jfce || _resource->jfce->fd < 0) { LOG(ERROR) << "Prepared URMA resource has no usable JFCE"; @@ -484,58 +500,57 @@ int UrmaEndpoint::AllocateResources() { if (ReqNotifyCq() != 0) { return -1; } - SocketOptions options; - options.user = this; - options.keytable_pool = _socket->keytable_pool(); - options.fd = _resource->jfce->fd; - options.on_edge_triggered_events = PollCq; - if (Socket::Create(options, &_cq_sid) < 0) { - PLOG(ERROR) << "Fail to create CQ socket"; + event_fd.reset(fcntl(_resource->jfce->fd, F_DUPFD_CLOEXEC, 0)); + if (event_fd < 0) { return -1; } - } else { - // Polling mode: synthetic carrier socket (no fd). - SocketOptions options; - options.user = this; - options.keytable_pool = _socket->keytable_pool(); - options.on_edge_triggered_events = PollCq; - if (Socket::Create(options, &_cq_sid) < 0) { - PLOG(ERROR) << "Fail to create CQ socket (polling)"; + } + std::unique_ptr user( + new (std::nothrow) UrmaCqSocketUser(_socket->id())); + if (!user) { + errno = ENOMEM; + return -1; + } + SocketOptions options; + options.keytable_pool = _socket->keytable_pool(); + options.on_edge_triggered_events = PollCq; + if (Socket::Create(options, &_cq_sid) != 0) { + PLOG(ERROR) << "Fail to create CQ socket"; + return -1; + } + // Install the owner before publishing the carrier to epoll or the poller. + SocketUniquePtr carrier; + CHECK_EQ(0, Socket::Address(_cq_sid, &carrier)); + carrier->_user = user.release(); + if (event_fd >= 0) { + if (carrier->ResetFileDescriptor(event_fd) != 0) { return -1; } + event_fd.release(); + } else { PollerAddCqSid(); } return 0; } void UrmaEndpoint::DeallocateResources() { - if (!_resource) { - return; - } - + std::unique_lock lock(_cq_mutex); if (FLAGS_urma_use_polling) { PollerRemoveCqSid(); } - // Tear down the CQ socket so the EventDispatcher stops calling PollCq. if (_cq_sid != INVALID_SOCKET_ID) { - SocketUniquePtr s; - if (Socket::Address(_cq_sid, &s) == 0) { - if (s->fd() >= 0) { - s->_io_event.RemoveConsumer(s->_fd); - } - s->_user = nullptr; // Do not release user (this UrmaEndpoint). - s->_fd = -1; // Already removed fd from epoll. - s->SetFailed(); - } + Socket::SetFailed(_cq_sid); _cq_sid = INVALID_SOCKET_ID; } - // Reusing a Jetty requires a driver-supported RESET plus a complete JFC - // drain. Until that lifecycle is implemented, prepared resources are - // one-shot: they accelerate connection setup but are destroyed on close. + // Prepared resources are one-shot. Destroy provider handles before + // releasing buffers referenced by outstanding work requests. delete _resource; _resource = nullptr; + _sbuf.clear(); + _rbuf.clear(); + _rbuf_data.clear(); } // ============================================================================ @@ -635,29 +650,20 @@ int UrmaEndpoint::ImportPeer(const ParsedHello& peer) { class UrmaIOBuf : private butil::IOBuf { friend class ::brpc::urma::UrmaEndpoint; public: - using butil::IOBuf::_ref_num; - using butil::IOBuf::_ref_at; - using butil::IOBuf::fetch1; - using butil::IOBuf::get_first_data_meta; - using butil::IOBuf::cutn; - // Build the SGE for the current head block. - // Returns bytes added, or -1 (errno set). - ssize_t cut_into_sglist(urma_sge_t* sglist, size_t* sge_index, - butil::IOBuf* to, size_t max_sge, - size_t max_len) { + // Describe the prefix without consuming it. The writer commits only + // after the provider accepts the WR. + ssize_t DescribePrefix(urma_sge_t* sglist, size_t* sge_index, + size_t max_sge, size_t max_len) const { size_t len = 0; - while (*sge_index < max_sge && len < max_len && _ref_num() != 0) { - butil::IOBuf::BlockRef const& r = _ref_at(0); - const void* start = fetch1(); + for (size_t i = 0; + i < _ref_num() && *sge_index < max_sge && len < max_len; ++i) { + const butil::IOBuf::BlockRef& r = _ref_at(i); + void* start = r.block->data + r.offset; urma_target_seg_t* tseg = - GetPoolSegFor(const_cast(start)); - if (!tseg) { - // User-registered memory: look up the seg handle. - uint64_t meta = get_first_data_meta(); - if (meta != 0) { - tseg = reinterpret_cast( - static_cast(meta)); - } + GetPoolSegFor(start); + if (!tseg && r.block->is_user_data()) { + tseg = reinterpret_cast( + static_cast(r.block->u.data_meta)); } if (!tseg) { errno = ERDMAMEM; @@ -679,7 +685,6 @@ class UrmaIOBuf : private butil::IOBuf { << (tseg->urma_ctx == GetUrmaContext()); return -1; } - cutn(to, this_len); len += this_len; (*sge_index)++; } @@ -709,7 +714,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ssize_t total_len = 0; while (current < ndata) { uint16_t remote_wnd = _remote_rq_window_size.load(butil::memory_order_relaxed); - uint16_t sq_wnd = _sq_window_size.load(butil::memory_order_relaxed); + uint16_t sq_wnd = _sq_window_size.load(butil::memory_order_acquire); if (remote_wnd == 0 || sq_wnd == 0) { if (total_len > 0) { break; @@ -727,24 +732,23 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { max_len = std::min(max_len, GetUrmaRecvBlockSize() - (URMA_SEND_BUFFER_ALIGNMENT - 1)); } - while (sge_index < static_cast(max_sge) && - this_len < max_len && current < ndata) { - auto* data = reinterpret_cast(from[current]); - if (data->empty()) { - ++current; - continue; - } - ssize_t n = - copy_send - ? static_cast(data->cutn(to, max_len - this_len)) - : data->cut_into_sglist(sglist, &sge_index, to, max_sge, - max_len - this_len); + for (size_t i = current; + i < ndata && this_len < max_len && + sge_index < static_cast(max_sge); ++i) { + const auto* data = reinterpret_cast(from[i]); + const ssize_t n = + copy_send ? static_cast( + std::min(from[i]->size(), max_len - this_len)) + : data->DescribePrefix(sglist, &sge_index, max_sge, + max_len - this_len); if (n < 0) { + to->clear(); if (errno == ERDMAMEM) { ReportFailure(UrmaFailureReason::ACCESS_ERROR); } return -1; } + from[i]->append_to(to, n); this_len += n; } if (this_len == 0) { @@ -755,6 +759,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { LOG(ERROR) << "Fail to prepare aligned URMA send buffer: length=" << this_len << " errno=" << errno; + to->clear(); ReportFailure(UrmaFailureReason::ACCESS_ERROR); return -1; } @@ -792,6 +797,11 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { const int provider_errno = errno; _remote_rq_window_size.fetch_add(1, butil::memory_order_relaxed); _sq_window_size.fetch_add(1, butil::memory_order_relaxed); + to->clear(); + if (rc == URMA_EAGAIN) { + errno = EAGAIN; + return total_len ? total_len : -1; + } LOG(WARNING) << "urma_post_jetty_send_wr failed: " << rc << ", provider_errno=" << provider_errno << " (" << berror(provider_errno) << ')' @@ -811,6 +821,13 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { errno = rc; return -1; } + // Commit the accepted prefix across the original write requests. + for (size_t remaining = this_len; remaining > 0;) { + remaining -= from[current]->pop_front(remaining); + if (from[current]->empty()) { + ++current; + } + } _sq_current = (_sq_current + 1) % (_sq_size - RESERVED_WR_NUM); total_len += static_cast(this_len); } @@ -1044,31 +1061,19 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { _sq_imm_window_size += 1; return SendAck(0); } - uint16_t wnd = 1; // We signal every WR (complete_enable=1). - uint16_t old = - _sq_window_size.load(butil::memory_order_relaxed); - while (true) { - if (old >= _local_window_capacity) { - LOG(WARNING) - << "URMA send completion exceeds SQ window: old=" << old - << " increment=" << wnd - << " capacity=" << _local_window_capacity - << " user_ctx=" << cr.user_ctx - << " on " << _socket->description(); - errno = EPROTO; - return -1; - } - if (_sq_window_size.compare_exchange_weak( - old, static_cast(old + wnd), - butil::memory_order_relaxed)) { - break; - } - } - for (uint16_t i = 0; i < wnd; ++i) { - _sbuf[_sq_sent].clear(); - _sq_sent = (_sq_sent + 1) % (_sq_size - RESERVED_WR_NUM); + const uint16_t old = _sq_window_size.load(butil::memory_order_relaxed); + if (old >= _local_window_capacity) { + LOG(WARNING) << "URMA send completion exceeds SQ window: old=" << old + << " capacity=" << _local_window_capacity + << " user_ctx=" << cr.user_ctx + << " on " << _socket->description(); + errno = EPROTO; + return -1; } - butil::subtle::MemoryBarrier(); + // Reclaim the buffer before making its slot available to the writer. + _sbuf[_sq_sent].clear(); + _sq_sent = (_sq_sent + 1) % (_sq_size - RESERVED_WR_NUM); + _sq_window_size.fetch_add(1, butil::memory_order_release); if (_remote_rq_window_size.load(butil::memory_order_relaxed) >= _local_window_capacity / 8) { _socket->WakeAsEpollOut(); @@ -1175,24 +1180,31 @@ void UrmaEndpoint::DispatchReceivedBytes(SocketUniquePtr& s, ssize_t bytes) { } void UrmaEndpoint::PollCq(Socket* m) { - auto* ep = static_cast(m->user()); - if (!ep || !ep->_resource || !ep->_resource->jfc) { + // Acquire the fd published by ResetFileDescriptor in event mode. + if (!FLAGS_urma_use_polling && + m->_fd.load(butil::memory_order_acquire) < 0) { return; } - SocketUniquePtr s; - if (Socket::Address(ep->_socket->id(), &s) != 0) { - return; - } - if (s->Failed()) { + const auto* user = static_cast(m->user()); + SocketUniquePtr owner; + if (!user || Socket::Address(user->owner_id, &owner) != 0) { return; } + auto* transport = static_cast(owner->_transport.get()); + transport->_urma_ep->DrainCq(m, owner); +} +void UrmaEndpoint::DrainCq(Socket* m, SocketUniquePtr& s) { const bool event_mode = !FLAGS_urma_use_polling; int progress = Socket::PROGRESS_INIT; while (true) { + std::unique_lock lock(_cq_mutex); + if (m->id() != _cq_sid || !_resource || !_resource->jfc || s->Failed()) { + return; + } urma_jfc_t* event_jfc = nullptr; if (event_mode) { - const int event_count = ep->WaitCqEvent(s, &event_jfc); + const int event_count = WaitCqEvent(s, &event_jfc); if (event_count < 0) { return; } @@ -1211,9 +1223,9 @@ void UrmaEndpoint::PollCq(Socket* m) { std::max(1, std::min(FLAGS_urma_cqe_poll_once, 32)); urma_cr_t crs[32]; const int cnt = - urma_poll_jfc(ep->_resource->jfc, n, crs); + urma_poll_jfc(_resource->jfc, n, crs); if (cnt < 0) { - ep->ReportFailure(UrmaFailureReason::CQ_POLL); + ReportFailure(UrmaFailureReason::CQ_POLL); return EIO; } if (cnt == 0) { @@ -1223,7 +1235,7 @@ void UrmaEndpoint::PollCq(Socket* m) { if (s->Failed()) { return ECANCELED; } - const ssize_t nr = ep->HandleCompletion(crs[i]); + const ssize_t nr = HandleCompletion(crs[i]); if (nr < 0) { return errno ? errno : EIO; } @@ -1241,7 +1253,7 @@ void UrmaEndpoint::PollCq(Socket* m) { uint32_t nevents = 1; urma_ack_jfc(&event_jfc, &nevents, 1); if (completion_error == 0) { - if (ep->ReqNotifyCq() != 0) { + if (ReqNotifyCq() != 0) { return; } @@ -1259,7 +1271,8 @@ void UrmaEndpoint::PollCq(Socket* m) { } return; } - ep->DispatchReceivedBytes(s, bytes); + lock.unlock(); + DispatchReceivedBytes(s, bytes); if (!event_mode) { return; @@ -1421,6 +1434,17 @@ void UrmaEndpoint::FallbackToTcp(UrmaTransport* transport, bool process_tcp) { } } +bool UrmaEndpoint::AllowNewUrmaConnection(const char* peer) { + if (!IsUrmaAvailable()) { + return false; + } + uint64_t epoch = 0; + const bool allowed = + GlobalUrmaRecoveryPolicy().AllowNewConnection(peer, &epoch); + _recovery_epoch.store(epoch, butil::memory_order_relaxed); + return allowed; +} + void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { return; @@ -1434,7 +1458,7 @@ void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { _socket->CreatedByConnect() ? butil::endpoint2str(_socket->remote_side()).c_str() : "", - _recovery_epoch, reason); + _recovery_epoch.load(butil::memory_order_relaxed), reason); } void UrmaEndpoint::FailHandshake(UrmaTransport* transport, int error, @@ -1463,9 +1487,8 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { SocketUniquePtr s(ep->_socket); auto* tp = static_cast(s->_transport.get()); UrmaConnect::RunGuard guard(static_cast(s->_app_connect.get())); - if (!IsUrmaAvailable() || !GlobalUrmaRecoveryPolicy().AllowNewConnection( - butil::endpoint2str(s->remote_side()).c_str(), - &ep->_recovery_epoch)) { + if (!ep->AllowNewUrmaConnection( + butil::endpoint2str(s->remote_side()).c_str())) { ep->FallbackToTcp(tp, true); return nullptr; } @@ -1517,7 +1540,8 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; - GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection( + ep->_recovery_epoch.load(butil::memory_order_relaxed)); ep->DispatchReceivedBytes(s, 0); ep->DrainEstablishedTcp(); return nullptr; @@ -1554,8 +1578,7 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->FailHandshake(tp, EPROTO, "invalid client hello"); return nullptr; } - if (!IsUrmaAvailable() || !GlobalUrmaRecoveryPolicy().AllowNewConnection( - "", &ep->_recovery_epoch)) { + if (!ep->AllowNewUrmaConnection("")) { tp->_urma_state = UrmaTransport::URMA_OFF; if (hs->SendLocalHello() < 0) { ep->FailHandshake(tp, errno ? errno : EIO, @@ -1616,7 +1639,7 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection( - ep->_recovery_epoch); + ep->_recovery_epoch.load(butil::memory_order_relaxed)); ep->DispatchReceivedBytes(s, 0); ep->DrainEstablishedTcp(); } else { @@ -1709,7 +1732,7 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) - << " recovery_epoch=" << _recovery_epoch + << " recovery_epoch=" << _recovery_epoch.load(butil::memory_order_relaxed) << " runtime_failure=" << UrmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index 033f7c9d45..90b2d724bf 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -147,7 +147,7 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { static void OnNewDataFromTcp(Socket* m); // CQ completion poller: the edge-triggered callback for the CQ socket - // (whose user is this endpoint). Drains urma_poll_jfc and feeds the + // (whose user holds the owning socket ID). Drains urma_poll_jfc and feeds the // completions into HandleCompletion, then calls InputMessenger. Event mode // also drains the aggregated JFCE until urma_wait_jfc reports no event, // then calls Socket::MoreReadEvents so later JFCE edges can start the @@ -214,6 +214,8 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Jetty, plus the CQ socket). Returns 0 on success. int AllocateResources(); void DeallocateResources(); + bool AllowNewUrmaConnection(const char* peer); + void DrainCq(Socket* carrier, SocketUniquePtr& owner); // Import the peer's jetty and buffer-pool segment. CRITICAL: // urma_import_seg is called BEFORE urma_import_jetty, otherwise the kernel @@ -277,7 +279,7 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Not owned. Socket* _socket; - uint64_t _recovery_epoch{0}; + butil::atomic _recovery_epoch{0}; butil::atomic _runtime_failure{UrmaFailureReason::NONE}; // Handshake state. @@ -293,6 +295,8 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // The SocketId wrapping the JFCE fd (event mode) or a synthetic carrier // (polling mode). PollCq is the edge-triggered callback on this socket. + // Serializes provider CQ operations with carrier registration and teardown. + butil::Mutex _cq_mutex; SocketId _cq_sid{INVALID_SOCKET_ID}; // ---- Send / recv window bookkeeping ---- diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index 47d9193f4c..69f6e2cc3e 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -445,6 +445,10 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, urma_target_seg_t* (*lookup)(void*)) { + if (!buffer || !sge || !lookup) { + errno = EINVAL; + return false; + } const size_t length = buffer->size(); const size_t capacity = GetUrmaRecvBlockSize(); if (length == 0 || capacity < URMA_SEND_BUFFER_ALIGNMENT || diff --git a/src/brpc/urma_transport.cpp b/src/brpc/urma_transport.cpp index c5cc163ffa..1484a9066f 100644 --- a/src/brpc/urma_transport.cpp +++ b/src/brpc/urma_transport.cpp @@ -80,7 +80,10 @@ int UrmaTransport::Reset(int32_t /*expected_nref*/) { } std::shared_ptr UrmaTransport::Connect() { - if (_default_connect == nullptr) { + // Child sockets inherit AppConnect options. The built-in handshake + // stores a pending callback and therefore belongs to one socket only. + if (!_default_connect || + dynamic_cast(_default_connect.get())) { return std::make_shared(); } return _default_connect; diff --git a/test/brpc_urma_mock_completion_unittest.cpp b/test/brpc_urma_mock_completion_unittest.cpp index 5a6173fb82..76cd2c2912 100644 --- a/test/brpc_urma_mock_completion_unittest.cpp +++ b/test/brpc_urma_mock_completion_unittest.cpp @@ -25,6 +25,35 @@ #if BRPC_WITH_URMA #include "urma_api.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_helper.h" + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static ssize_t Send(Socket* socket, urma_jetty_t* jetty, + urma_target_jetty_t* target, butil::IOBuf** buffers) { + UrmaEndpoint endpoint(socket); + UrmaResource resource; + resource.jetty = jetty; + resource.remote_jetty = target; + endpoint._resource = &resource; + endpoint._sbuf.resize(2); + endpoint._sq_window_size.store(4, butil::memory_order_relaxed); + endpoint._remote_rq_window_size.store(4, butil::memory_order_relaxed); + endpoint._remote_recv_block_size = 4; + const ssize_t sent = endpoint.CutFromIOBufList(buffers, 2); + endpoint._resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return sent; + } +}; + +} // namespace urma +} // namespace brpc namespace { @@ -94,6 +123,75 @@ class UrmaMockCompletionTest : public testing::Test { urma_target_jetty_t* _target = nullptr; }; +TEST_F(UrmaMockCompletionTest, undeliverable_send_has_no_success_completion) { + urma_jfs_wr_t send{}; + send.opcode = URMA_OPC_SEND; + send.flag.bs.complete_enable = 1; + send.tjetty = _target; + urma_jfs_wr_t* bad = nullptr; + urma_cr_t cr{}; + + EXPECT_EQ(URMA_EAGAIN, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); + + ASSERT_EQ(URMA_SUCCESS, urma_delete_jfc(_recv_cq)); + _recv_cq = nullptr; + bad = nullptr; + EXPECT_NE(URMA_SUCCESS, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); + + ASSERT_EQ(URMA_SUCCESS, urma_delete_jetty(_receiver)); + _receiver = nullptr; + bad = nullptr; + EXPECT_NE(URMA_SUCCESS, urma_post_jetty_send_wr(_sender, &send, &bad)); + EXPECT_EQ(&send, bad); + EXPECT_EQ(0, urma_poll_jfc(_send_cq, 1, &cr)); +} + +TEST_F(UrmaMockCompletionTest, provider_backpressure_keeps_unsent_suffix) { + brpc::SocketId sid; + ASSERT_EQ(0, brpc::Socket::Create(brpc::SocketOptions(), &sid)); + brpc::SocketUniquePtr socket; + ASSERT_EQ(0, brpc::Socket::Address(sid, &socket)); + + char payload[] = "abcdefghijk"; + urma_target_seg_t segment{}; + segment.urma_ctx = brpc::urma::GetUrmaContext(); + segment.seg.ubva.va = reinterpret_cast(payload); + segment.seg.len = sizeof(payload) - 1; + const auto meta = reinterpret_cast(&segment); + butil::IOBuf head; + butil::IOBuf tail; + ASSERT_EQ(0, head.append_user_data_with_meta(payload, 6, [](void*) {}, meta)); + ASSERT_EQ(0, tail.append_user_data_with_meta(payload + 6, 5, + [](void*) {}, meta)); + butil::IOBuf* buffers[] = {&head, &tail}; + + char received[4]{}; + urma_sge_t sge{}; + sge.addr = reinterpret_cast(received); + sge.len = sizeof(received); + urma_jfr_wr_t recv{}; + recv.src.sge = &sge; + recv.src.num_sge = 1; + urma_jfr_wr_t* bad = nullptr; + ASSERT_EQ(URMA_SUCCESS, urma_post_jfr_wr(_recv_queue, &recv, &bad)); + + EXPECT_EQ(4, brpc::urma::UrmaRecoveryTestPeer::Send( + socket.get(), _sender, _target, buffers)); + EXPECT_EQ("abcd", std::string(received, sizeof(received))); + EXPECT_EQ("ef", head.to_string()); + EXPECT_EQ("ghijk", tail.to_string()); + EXPECT_EQ(-1, brpc::urma::UrmaRecoveryTestPeer::Send( + socket.get(), _sender, _target, buffers)); + EXPECT_EQ(EAGAIN, errno); + EXPECT_EQ("ef", head.to_string()); + EXPECT_EQ("ghijk", tail.to_string()); + socket->SetFailed(); +} + TEST_F(UrmaMockCompletionTest, send_completion_follows_payload_delivery) { // Keep the copy large enough for a concurrent CQ consumer to run while // the mock is still using the source buffer. diff --git a/test/brpc_urma_runtime_unittest.cpp b/test/brpc_urma_runtime_unittest.cpp index fac5b91fbf..cd6cc998b9 100644 --- a/test/brpc_urma_runtime_unittest.cpp +++ b/test/brpc_urma_runtime_unittest.cpp @@ -242,6 +242,9 @@ TEST(UrmaRuntimeHelperTest, copied_send_failure_preserves_source_payload) { urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); EXPECT_EQ(ERDMAMEM, errno); EXPECT_EQ("unchanged", buffer.to_string()); + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, nullptr)); + EXPECT_EQ(EINVAL, errno); + EXPECT_EQ("unchanged", buffer.to_string()); const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); buffer.clear(); buffer.append(large); @@ -299,7 +302,9 @@ class UrmaRecoveryTestPeer { return ep->ImportPeer(peer); } static void Establish(UrmaEndpoint* ep) { - GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); + uint64_t epoch = 0; + GlobalUrmaRecoveryPolicy().AllowNewConnection("", &epoch); + ep->_recovery_epoch.store(epoch, butil::memory_order_relaxed); ep->_state = UrmaEndpoint::ESTABLISHED; } static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { @@ -316,14 +321,18 @@ class UrmaRecoveryTestPeer { resource.jfc = &invalid_jfc; ep->_resource = &resource; SocketOptions options; - options.user = ep; SocketId sid; if (Socket::Create(options, &sid) == 0) { SocketUniquePtr carrier; if (Socket::Address(sid, &carrier) == 0) { const bool polling = FLAGS_urma_use_polling; FLAGS_urma_use_polling = true; - UrmaEndpoint::PollCq(carrier.get()); + SocketUniquePtr owner; + if (Socket::Address(ep->_socket->id(), &owner) == 0) { + ep->_cq_sid = sid; + ep->DrainCq(carrier.get(), owner); + ep->_cq_sid = INVALID_SOCKET_ID; + } FLAGS_urma_use_polling = polling; carrier->SetFailed(); }