From 45e08065d0d9b33cfef2f9834d3bfb6c0fff53f1 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:22:22 -0700 Subject: [PATCH 1/8] Design RDMA runtime recovery and controlled TCP fallback --- docs/cn/rdma.md | 5 + docs/cn/rdma_runtime_recovery.md | 231 +++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 docs/cn/rdma_runtime_recovery.md diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index fd70686bb1..96fa00a103 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,3 +1,8 @@ +# 运行期故障恢复设计 + +[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md) 为尚未实现的设计草案， +讨论端口故障隔离、冷却与恢复探测、运行时降级控制及多端口主备。 + # 编译 由于RDMA对驱动与硬件有要求，目前仅支持在Linux系统编译并运行RDMA功能。 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md new file mode 100644 index 0000000000..8e1af1ef47 --- /dev/null +++ b/docs/cn/rdma_runtime_recovery.md @@ -0,0 +1,231 @@ + + +# RDMA 运行期故障恢复与 TCP 降级设计 + +状态：设计草案，尚未实现。本文中的接口、参数和指标均为提案。 + +基线为 LinQuickDev/brpc master 的 +`87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 +本提案改变后续连接的传输选择；不承诺存量 RPC 无损迁移或 exactly-once。 + +## 问题与源码依据 + +当前握手可以回落 TCP，但 RDMA 建立后的故障主要通过连接失败处理： + +| 位置 | 当前行为 | 需要补充的能力 | +| --- | --- | --- | +| `RdmaConnect::StartConnect`、握手状态机 | RDMA 不可用或协商失败时回落 TCP | 后续建链能记住运行期故障并绕过故障路径 | +| `RdmaEndpoint::PollCq` | 错误 CQE 调用 `Socket::SetFailed` | 错误分类、隔离和恢复探测 | +| `OnRdmaAsyncEvent` | PORT_ERR / PORT_ACTIVE 只确认事件 | 端口状态与连接准入联动 | +| `GlobalDisableRdma` | 修改可用状态并记录 FATAL | 独立、非致命的运维降级控制 | +| `CutFromIOBufList` | 窗口耗尽返回 EAGAIN；post_send 失败按不可恢复处理 | 保留背压语义，检测有未完成发送时的长期无进展 | +| `BringUpQp` | timeout=14、retry_cnt=7、rnr_retry=0 | 明确错误语义及受限配置，而非把所有错误归为超时 | + +源码分别位于 `src/brpc/rdma/rdma_endpoint.cpp`、 +`src/brpc/rdma/rdma_helper.cpp` 和 `src/brpc/rdma_transport.cpp`。 +RPC deadline、verbs ACK 重试耗尽、RNR 重试耗尽和本地 SQ 背压是不同事件。 + +## 目标和兼容性 + +1. 本地 RDMA 端口故障后，快速停止该端口的新建链并使相关连接失败。 +2. 后续连接优先使用可达的备用 RDMA 路径，无可用路径时通过新 TCP 连接恢复。 +3. 对故障路径设置冷却和有界恢复探测，避免每次 RPC 都重新尝试故障 RDMA。 +4. 提供运行时 TCP-only 开关，并明确存量连接的处理方式。 +5. 给出 SQ 停顿、RNR 和链路错误的独立策略、指标和测试标准。 + +自动恢复默认关闭；启用前保留现有单设备、单端口和握手协议行为。 +TCP fallback 遵循现有协议与 SSL 限制，不绕过认证，不隐式降低安全配置。 +发布第一阶段只实现单设备故障隔离和 TCP 恢复；跨 HCA 主备在资源模型改造后提供。 + +## 恢复边界 + +故障连接执行正常 SetFailed、唤醒写等待者并进入回收流程。 +策略先发布故障状态，再传播连接失败，使同时发生的新建链看到最新路由。 +已经建立的 RDMA 连接不改为在原 TCP fd 上发送 RPC 数据。 +该 fd 在握手后有独立语义，直接切换会造成消息边界、解析状态和顺序问题。 + +只由现有 Controller、RetryPolicy、max_retry 和 RPC deadline 决定是否重试。 +新增策略不增加重试配额、不重置 deadline、不自动重放已 post 的 WR。 +请求可能已被服务端执行时保留现有错误语义；有副作用的业务仍需幂等键或去重。 +恢复策略只保证后续连接避开故障 RDMA，不保证正在执行的调用成功。 + +## 组件与资源所有权 + +在 `src/brpc/rdma/` 新增 `RdmaRecoveryPolicy`，管理准入、冷却和探测。 +`RdmaTransport` 在建链前获取一次不可变的 route lease； +`RdmaEndpoint` 的 QP、CQ、GID、port 和注册内存绑定到该 lease。 + +状态分为两个作用域： + +- 本地端口键：device identity、port number、资源 generation。 +- 对端路径键：本地端口键、远端地址、传输模式、协商相关身份。 + +QP 错误先影响连接或对端路径，不根据一个对端失败就禁用整块网卡。 +新连接可读取端口状态和对端状态的不可变快照；数据发送热路径不加全局锁。 +有界缓存使用过期时间和容量限制；只逐出无活动 lease、无探测的对端状态。 +端口配置固定在启动时，运维开关在运行时更新策略 epoch。 + +## 状态机与故障分类 + +端口和对端路径分别维护 HEALTHY、COOLDOWN、PROBING、DISABLED。 +单调时钟驱动冷却；每个状态更新带 generation 和 policy epoch。 + +| 输入 | 状态变化 | 行为 | +| --- | --- | --- | +| 本地 PORT_ERR | 端口进入 COOLDOWN | 禁止新 RDMA；使绑定该 generation 的连接失败 | +| PORT_ACTIVE | 保留隔离，记录链路恢复 | 不立即恢复全量流量，等待冷却和探测 | +| QP/CQ 错误 | 连接失败，必要时对端路径进入 COOLDOWN | 不据此断定本地端口故障 | +| RETRY_EXC | 对端路径进入 COOLDOWN | 原因可能是路径、对端或网络；不累计为本地端口故障 | +| RNR_RETRY_EXC | 对端路径进入 COOLDOWN | 记录接收资源/流控异常，不切坏整个本地端口 | +| 正常 SQ/RQ 窗口为零 | 不改变健康状态 | EAGAIN 背压，等待 completion / credit | +| post_send 失败 | 连接失败，按 errno 分类 | 保存原始错误；不直接重投 WR 或循环忙重试 | +| 运维 TCP-only | 准入进入 DISABLED | 新建链直接选择 TCP，按下文处理存量连接 | +| DEVICE_FATAL | 对应资源域永久 DISABLED | 仅在安全排空得到验证后支持非致命降级，否则保留现有 FATAL | + +禁止把 CQ flush 连锁错误重复计为多次独立故障。 +同一 socket generation 的失败使用一次性报告，过期事件不得修改新一代状态。 +内存注册错误、远端访问错误和本地保护错误保留独立诊断；不以自动重试掩盖编程错误。 + +## 冷却与恢复探测 + +首次隔离使用基础冷却时间，连续探测失败指数退避至上限，并加入有界 jitter。 +冷却期所有新连接跳过该路径，选择健康候选路径或 TCP。 +冷却结束不等于恢复：原子准入最多一个新 RDMA 连接作为探测，其余请求继续走 TCP。 +PORT_ERR 后还必须先确认本地端口 ACTIVE，才能发放探测 lease。 + +握手成功只能证明建链成功。探测连接收到有效 RDMA 数据、且至少一次发送获得完成后， +才把路径恢复为 HEALTHY；不能以 TCP 成功证明 RDMA 已恢复。 +不构造会产生业务副作用的探测 RPC；使用正常业务流量并受原调用 deadline 约束。 +探测在预算内失败或没有足够证据时重新冷却，不长期占用探测名额。 +一条探测连接同时持有端口和对端 lease，成功/失败只释放对应 epoch 的名额。 + +## 运行时控制与连接复用 + +提议 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 及进程级原子 setter/getter。 +setter 校验输入、增加 epoch，并唤醒控制任务；不直接销毁 QP 或 MR。 +运维系统可调用 API，初版不新增可远程修改状态的 HTTP 接口。 + +TCP_ONLY 的完成语义是“停止新 RDMA 准入并返回新策略 epoch”。 +它不表示所有旧连接已排空；单独暴露旧 epoch 的活动连接数。 +默认允许健康存量 RPC 完成，之后将连接退休并阻止新 RPC 复用。 +对于持续复用或长连接，必须在请求获取 socket 时检查退休位，不能只在握手时检查。 +可选强制模式使旧连接失败，但不自动重放其中请求；其影响须由调用方显式选择。 +重新切回 AUTO 后通过 PROBING 恢复，不瞬间全量重建 RDMA 连接。 + +SocketMap / 连接池的复用键需区分路由模式、端口资源域和 policy epoch。 +同一逻辑 Channel 下，失败的 RDMA socket、临时 TCP socket 和探测 socket 不得误复用。 +TCP fallback 必须创建明确的 TCP transport/socket，不再进入 RDMA 初始化和握手。 +现有 socket 复活路径重新获取 lease，不把旧 endpoint 的路由状态带到新 generation。 +本地 RDMA 故障不得直接把仍可经 TCP 访问的服务实例长期标记为不可用； +需要验证 SetFailed 与健康检查、负载均衡惩罚的现有交互，保持单次 RPC 的错误通知。 + +## 多端口主备与注册内存 + +候选路径启动时配置并验证；顺序选择健康主路径，再选择健康备用路径。 +不在单个 QP 上修改活动路径，也不把旧 QP 搬到另一 HCA。 +切换意味着建立新 QP，重新交换正确的 GID/QPN 等信息。 +本地备用端口不一定能访问同一个对端，探测失败应隔离该对端路径。 + +当前全局 context / PD / MR / lkey 模型不能直接支撑跨 HCA 切换。 +跨 HCA 必须先引入 `RdmaDeviceContext`：每个 context 拥有 PD、事件源及 MR 注册表， +IOBuf 内存块按目标 PD 查找 lkey，不能把主设备的 lkey 用于备用设备。 +预注册备用设备需要容量预算及失败回滚；用户注册内存也需按设备注册和保活。 +不能安全注册到备用 PD 的 buffer 走明确的受控复制路径或 TCP，不能使用无效 lkey。 +同 HCA 不同端口若共享 PD 仍需验证 GID、路由和对端可达性；不默认等同于跨 HCA。 + +## SQ 停顿和 RNR 策略 + +SQ 窗口耗尽保留 EAGAIN。可选 watchdog 只跟踪有 outstanding WR 或待发送且被窗口阻塞的连接。 +使用单调时间记录最后发送完成及 credit 前进，不以 idle 时长判断故障。 +待发送数据、未完成 WR 和 credit 的状态分别记录，避免接收数据持续到达掩盖发送停顿。 +超过无进展阈值后使该连接失败并隔离对端路径；不导致整个端口隔离或进程退出。 +RPC 超时与传输 watchdog 独立，后者不延长任何调用的 deadline。 + +ACK timeout 指数、retry count 和 RNR retry 保持现有值作为默认，分别校验 verbs 范围。 +首版不提供无限 RNR 重试：它会把错误变成无法及时检测的停顿。 +允许的有限 RNR 重试需结合收包 buffer 补充延迟测试；调参不能替代滑动窗口正确性。 +watchdog 默认关闭，硬件和拥塞测试确定阈值后再建议生产开启。 + +## 提议配置与可观测性 + +以下名称尚未实现。配置启动时校验，运行时只允许通过专用 API 更新恢复模式。 + +| 参数 | 初始建议 | 约束 | +| --- | --- | --- | +| rdma_recovery_mode | legacy | legacy / auto / tcp_only；auto 显式启用 | +| rdma_path_cooldown_ms | 5000 | 正数，使用单调时钟 | +| rdma_path_max_cooldown_ms | 60000 | 不小于基础冷却时间 | +| rdma_recovery_probe_timeout_ms | 1000 | 正数；探测调用仍遵守更短的原 deadline | +| rdma_send_stall_timeout_ms | 0 | 0 关闭；启用时必须为正数 | +| rdma_backup_paths | 空 | 启动时配置；仅在设备资源域改造后支持 | + +暴露连接失败原因、RDMA-to-TCP 新建链次数、隔离次数、探测成功/失败、 +端口状态、旧 epoch 活动连接数、无进展发送时长及 SQ/RQ credit。 +指标避免使用无界远端地址作为 bvar 名称；对端细节放入有界诊断视图。 +日志包含 socket generation、device、port、WC status/vendor_err、errno 和策略 epoch， +对相同故障做限频，并保留首次根因，避免被 WR_FLUSH 覆盖。 + +## 并发、回收与停机 + +控制任务只持有弱 socket 标识和带代际的 route lease；执行时重新寻址并校验 generation。 +状态发布使用明确的 acquire/release 配对；事件回调不在持有策略锁时调用 SetFailed。 +防止 SetFailed、复活、探测定时器和端口事件之间形成锁顺序循环。 +故障后先停止准入和 WR 提交，再停止对应 CQ 处理并按现有安全流程 reset/drain QP/CQ， +最后释放发送 buffer、MR、PD 和 context。完成状态未知的 DMA 内存不能提前复用。 +停机取消探测和 watchdog、等待回调退出，再销毁设备资源。 +DEVICE_FATAL 的资源回收必须单独验证；无法证明安全排空时不移除当前 FATAL 保护。 + +## 分阶段交付 + +1. 单设备策略、错误分类、端口事件、冷却/探测、TCP 新连接恢复及受控运行时模式。 + 包含 SocketMap/连接池退休与复用边界、代际隔离和基本指标。 +2. 有界发送无进展检测、有限 RNR 参数校验与故障注入；默认保持现有背压行为。 +3. 多设备资源域、按 PD 注册内存及候选路径选择，完成后开放多端口/跨 HCA 主备。 + +每阶段单独实现和验证；阶段一不宣称支持跨 HCA 或 DEVICE_FATAL 无损恢复。 +本设计 PR 只增加文档；后续实现 PR 才启用功能和增加对应测试。 + +## 验证和验收标准 + +| 层级 | 场景 | 验收条件 | +| --- | --- | --- | +| 策略单测，注入时钟 | 冷却、退避、epoch 变化、并发准入 | 冷却零 RDMA 准入；到期最多一个探测；旧结果无效 | +| 连接测试，mock verbs | PORT_ERR 与并发建链、复活竞争 | 受影响连接失败并唤醒等待者；后续连接使用 TCP | +| 错误分类 | RNR、RETRY_EXC、WR_FLUSH、post_send errno | RNR 不隔离本地端口；连锁 flush 不重复计数 | +| 背压测试 | SQ/RQ 为零后 CQ/credit 恢复 | EAGAIN 可恢复，无误降级、忙循环或丢失唤醒 | +| watchdog | idle、持续接收但发送停顿、慢恢复 | idle 不触发；真实发送停顿有界失败；恢复更新进度 | +| RPC 语义 | 部分 post 后失败、服务端已执行 | 不在 transport 中重放；deadline 和重试配额不增加 | +| 复用与开关 | TCP_ONLY、AUTO、旧池连接、持续流量 | 新请求不复用退休 RDMA；TCP 成功不算 RDMA 探测成功 | +| 兼容性 | 开关关闭、新旧对端组合 | legacy 行为及现有握手格式不变 | +| 生命周期 | reset/revive、晚到 CQ 回调、停机 | 无 UAF、重复回收、提前释放 DMA buffer 或停机挂起 | +| 双端 Linux RDMA | 拔链路、恢复链路、接收资源不足 | 后续连接可走 TCP；恢复仅经探测；无重连风暴 | +| 双 HCA Linux RDMA | 主路径故障、不同 PD 的 lkey、注册失败 | 备用 QP 使用匹配 MR；失败可退 TCP；无无效 DMA | + +mock 验证不能代替真机验证。硬件验收记录 RDMA provider、固件、拓扑、 +RPC 成功率及恢复时延分布；使用受控 RoCE/IB 测试环境完成端口故障注入。 +分别测量关闭特性、健康 AUTO、冷却 TCP 三种情况下的吞吐、P99 和 CPU。 +合入实现前需给出恢复预算及性能回归容限；本草案不提供未经测量的性能承诺。 + +## 待评审决策 + +- 首阶段运行时控制是否采用进程级 API，后续是否需要 Channel 级覆盖？ +- 长连接退休的默认排空期限和强制失败入口如何与现有连接池接口对齐？ +- 多 HCA 用户内存注册是否提供显式 registration set，还是通过逐设备查询接口？ +- 发送停顿阈值及允许的有限 RNR 重试是否需要按部署配置档提供建议？ From 88ece2db9bf84e4bda79070bd4d90e0a6a84b2d0 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:28:05 -0700 Subject: [PATCH 2/8] Simplify RDMA recovery design --- docs/cn/rdma.md | 4 +- docs/cn/rdma_runtime_recovery.md | 271 +++++++++---------------------- 2 files changed, 83 insertions(+), 192 deletions(-) diff --git a/docs/cn/rdma.md b/docs/cn/rdma.md index 96fa00a103..052fbc13d0 100644 --- a/docs/cn/rdma.md +++ b/docs/cn/rdma.md @@ -1,7 +1,7 @@ # 运行期故障恢复设计 -[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md) 为尚未实现的设计草案， -讨论端口故障隔离、冷却与恢复探测、运行时降级控制及多端口主备。 +端口故障隔离、冷却探测、运行时降级及多端口主备方案见 +[RDMA 运行期故障恢复与 TCP 降级设计](rdma_runtime_recovery.md)。 # 编译 diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index 8e1af1ef47..d9621406c4 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -19,213 +19,104 @@ under the License. # RDMA 运行期故障恢复与 TCP 降级设计 -状态：设计草案，尚未实现。本文中的接口、参数和指标均为提案。 +设计提案，基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 -基线为 LinQuickDev/brpc master 的 -`87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 -本提案改变后续连接的传输选择；不承诺存量 RPC 无损迁移或 exactly-once。 +## 当前问题 -## 问题与源码依据 +RDMA 握手阶段支持回落 TCP，建链后的错误 CQE 主要通过 `Socket::SetFailed` 处理。 +`PORT_ERR` / `PORT_ACTIVE` 事件仅做确认，缺少端口隔离、冷却和恢复探测。 +`GlobalDisableRdma` 使用 FATAL，需要独立的运行时降级入口。 -当前握手可以回落 TCP，但 RDMA 建立后的故障主要通过连接失败处理： +相关代码：`src/brpc/rdma/rdma_endpoint.cpp`、`src/brpc/rdma/rdma_helper.cpp`、 +`src/brpc/rdma_transport.cpp`。 -| 位置 | 当前行为 | 需要补充的能力 | -| --- | --- | --- | -| `RdmaConnect::StartConnect`、握手状态机 | RDMA 不可用或协商失败时回落 TCP | 后续建链能记住运行期故障并绕过故障路径 | -| `RdmaEndpoint::PollCq` | 错误 CQE 调用 `Socket::SetFailed` | 错误分类、隔离和恢复探测 | -| `OnRdmaAsyncEvent` | PORT_ERR / PORT_ACTIVE 只确认事件 | 端口状态与连接准入联动 | -| `GlobalDisableRdma` | 修改可用状态并记录 FATAL | 独立、非致命的运维降级控制 | -| `CutFromIOBufList` | 窗口耗尽返回 EAGAIN；post_send 失败按不可恢复处理 | 保留背压语义，检测有未完成发送时的长期无进展 | -| `BringUpQp` | timeout=14、retry_cnt=7、rnr_retry=0 | 明确错误语义及受限配置，而非把所有错误归为超时 | +## 恢复流程 -源码分别位于 `src/brpc/rdma/rdma_endpoint.cpp`、 -`src/brpc/rdma/rdma_helper.cpp` 和 `src/brpc/rdma_transport.cpp`。 -RPC deadline、verbs ACK 重试耗尽、RNR 重试耗尽和本地 SQ 背压是不同事件。 +故障检测 → 隔离故障路径 → 新连接选择备用 RDMA 或 TCP → 冷却 → 单连接探测 → 恢复。 -## 目标和兼容性 +新增 `RdmaRecoveryPolicy`，分别管理本地端口和对端路径的健康状态。 +本地端口以 device、port 标识，对端路径增加远端地址。 +`RdmaTransport` 建链时获取路由快照，`RdmaEndpoint` 持有对应设备资源。 -1. 本地 RDMA 端口故障后，快速停止该端口的新建链并使相关连接失败。 -2. 后续连接优先使用可达的备用 RDMA 路径，无可用路径时通过新 TCP 连接恢复。 -3. 对故障路径设置冷却和有界恢复探测，避免每次 RPC 都重新尝试故障 RDMA。 -4. 提供运行时 TCP-only 开关，并明确存量连接的处理方式。 -5. 给出 SQ 停顿、RNR 和链路错误的独立策略、指标和测试标准。 +发生故障时，先发布隔离状态，再使受影响连接失败并唤醒写等待者。 +后续连接按主 RDMA、备用 RDMA、TCP 的顺序选择可用路径。 +TCP 降级使用新连接；存量 RPC 交给现有 Controller 重试策略处理，沿用原配额和 deadline。 -自动恢复默认关闭；启用前保留现有单设备、单端口和握手协议行为。 -TCP fallback 遵循现有协议与 SSL 限制，不绕过认证，不隐式降低安全配置。 -发布第一阶段只实现单设备故障隔离和 TCP 恢复；跨 HCA 主备在资源模型改造后提供。 +## 状态与错误处理 -## 恢复边界 +路径状态为 HEALTHY、COOLDOWN、PROBING、DISABLED。 -故障连接执行正常 SetFailed、唤醒写等待者并进入回收流程。 -策略先发布故障状态，再传播连接失败，使同时发生的新建链看到最新路由。 -已经建立的 RDMA 连接不改为在原 TCP fd 上发送 RPC 数据。 -该 fd 在握手后有独立语义，直接切换会造成消息边界、解析状态和顺序问题。 +| 事件 | 处理 | +| --- | --- | +| PORT_ERR | 端口进入 COOLDOWN，关闭该端口上的连接 | +| PORT_ACTIVE | 标记端口可探测，冷却结束后进入 PROBING | +| QP/CQ 错误、RETRY_EXC | 连接失败并隔离对应对端路径 | +| RNR_RETRY_EXC | 隔离对端路径，记录接收资源或流控异常 | +| SQ/RQ 窗口耗尽 | 返回 EAGAIN，等待 completion 或 credit | +| post_send 失败 | 保存 errno，使连接失败并按错误类型更新路径状态 | +| DEVICE_FATAL | 禁用对应设备资源域，沿用现有 FATAL 处理 | -只由现有 Controller、RetryPolicy、max_retry 和 RPC deadline 决定是否重试。 -新增策略不增加重试配额、不重置 deadline、不自动重放已 post 的 WR。 -请求可能已被服务端执行时保留现有错误语义；有副作用的业务仍需幂等键或去重。 -恢复策略只保证后续连接避开故障 RDMA，不保证正在执行的调用成功。 +冷却使用单调时钟，连续失败采用指数退避和 jitter。 +到期后最多准入一个 RDMA 探测连接，其余新连接继续走备用路径或 TCP。 +探测使用正常业务流量；收到有效 RDMA 数据且发送获得完成后恢复 HEALTHY， +失败或探测超时则重新冷却。 -## 组件与资源所有权 +每次状态更新携带 generation 和 policy epoch。 +同一连接代际只报告一次故障，晚到事件按代际过滤。 -在 `src/brpc/rdma/` 新增 `RdmaRecoveryPolicy`，管理准入、冷却和探测。 -`RdmaTransport` 在建链前获取一次不可变的 route lease； -`RdmaEndpoint` 的 QP、CQ、GID、port 和注册内存绑定到该 lease。 +## 运行时降级 -状态分为两个作用域： +提供进程级 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 和 setter/getter。 +LEGACY 为默认值；AUTO 启用恢复策略；TCP_ONLY 将后续请求切到 TCP。 -- 本地端口键：device identity、port number、资源 generation。 -- 对端路径键：本地端口键、远端地址、传输模式、协商相关身份。 +模式更新递增 policy epoch。TCP_ONLY 下，健康存量 RPC 完成后连接退休； +请求获取 socket 时检查退休状态，新请求使用新连接。 +切回 AUTO 后进入 PROBING，逐步恢复 RDMA。 -QP 错误先影响连接或对端路径，不根据一个对端失败就禁用整块网卡。 -新连接可读取端口状态和对端状态的不可变快照；数据发送热路径不加全局锁。 -有界缓存使用过期时间和容量限制；只逐出无活动 lease、无探测的对端状态。 -端口配置固定在启动时，运维开关在运行时更新策略 epoch。 +SocketMap 和连接池按传输模式、设备资源域和 epoch 区分连接。 +socket 复活时重新获取路由快照。服务实例的健康检查与 RDMA 路径状态分开维护。 -## 状态机与故障分类 +## 多端口主备 -端口和对端路径分别维护 HEALTHY、COOLDOWN、PROBING、DISABLED。 -单调时钟驱动冷却；每个状态更新带 generation 和 policy epoch。 +启动时配置主备路径，切换时创建新 QP 并交换对应 GID/QPN。 +同 HCA 多端口共享 PD 时复用注册内存，各端口独立维护路由和健康状态。 -| 输入 | 状态变化 | 行为 | -| --- | --- | --- | -| 本地 PORT_ERR | 端口进入 COOLDOWN | 禁止新 RDMA；使绑定该 generation 的连接失败 | -| PORT_ACTIVE | 保留隔离，记录链路恢复 | 不立即恢复全量流量，等待冷却和探测 | -| QP/CQ 错误 | 连接失败，必要时对端路径进入 COOLDOWN | 不据此断定本地端口故障 | -| RETRY_EXC | 对端路径进入 COOLDOWN | 原因可能是路径、对端或网络；不累计为本地端口故障 | -| RNR_RETRY_EXC | 对端路径进入 COOLDOWN | 记录接收资源/流控异常，不切坏整个本地端口 | -| 正常 SQ/RQ 窗口为零 | 不改变健康状态 | EAGAIN 背压，等待 completion / credit | -| post_send 失败 | 连接失败，按 errno 分类 | 保存原始错误；不直接重投 WR 或循环忙重试 | -| 运维 TCP-only | 准入进入 DISABLED | 新建链直接选择 TCP，按下文处理存量连接 | -| DEVICE_FATAL | 对应资源域永久 DISABLED | 仅在安全排空得到验证后支持非致命降级，否则保留现有 FATAL | - -禁止把 CQ flush 连锁错误重复计为多次独立故障。 -同一 socket generation 的失败使用一次性报告，过期事件不得修改新一代状态。 -内存注册错误、远端访问错误和本地保护错误保留独立诊断；不以自动重试掩盖编程错误。 - -## 冷却与恢复探测 - -首次隔离使用基础冷却时间，连续探测失败指数退避至上限，并加入有界 jitter。 -冷却期所有新连接跳过该路径，选择健康候选路径或 TCP。 -冷却结束不等于恢复：原子准入最多一个新 RDMA 连接作为探测，其余请求继续走 TCP。 -PORT_ERR 后还必须先确认本地端口 ACTIVE，才能发放探测 lease。 - -握手成功只能证明建链成功。探测连接收到有效 RDMA 数据、且至少一次发送获得完成后， -才把路径恢复为 HEALTHY；不能以 TCP 成功证明 RDMA 已恢复。 -不构造会产生业务副作用的探测 RPC；使用正常业务流量并受原调用 deadline 约束。 -探测在预算内失败或没有足够证据时重新冷却，不长期占用探测名额。 -一条探测连接同时持有端口和对端 lease，成功/失败只释放对应 epoch 的名额。 - -## 运行时控制与连接复用 - -提议 `RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }` 及进程级原子 setter/getter。 -setter 校验输入、增加 epoch，并唤醒控制任务；不直接销毁 QP 或 MR。 -运维系统可调用 API，初版不新增可远程修改状态的 HTTP 接口。 - -TCP_ONLY 的完成语义是“停止新 RDMA 准入并返回新策略 epoch”。 -它不表示所有旧连接已排空；单独暴露旧 epoch 的活动连接数。 -默认允许健康存量 RPC 完成，之后将连接退休并阻止新 RPC 复用。 -对于持续复用或长连接，必须在请求获取 socket 时检查退休位，不能只在握手时检查。 -可选强制模式使旧连接失败，但不自动重放其中请求；其影响须由调用方显式选择。 -重新切回 AUTO 后通过 PROBING 恢复，不瞬间全量重建 RDMA 连接。 - -SocketMap / 连接池的复用键需区分路由模式、端口资源域和 policy epoch。 -同一逻辑 Channel 下，失败的 RDMA socket、临时 TCP socket 和探测 socket 不得误复用。 -TCP fallback 必须创建明确的 TCP transport/socket，不再进入 RDMA 初始化和握手。 -现有 socket 复活路径重新获取 lease，不把旧 endpoint 的路由状态带到新 generation。 -本地 RDMA 故障不得直接把仍可经 TCP 访问的服务实例长期标记为不可用； -需要验证 SetFailed 与健康检查、负载均衡惩罚的现有交互，保持单次 RPC 的错误通知。 - -## 多端口主备与注册内存 - -候选路径启动时配置并验证；顺序选择健康主路径，再选择健康备用路径。 -不在单个 QP 上修改活动路径，也不把旧 QP 搬到另一 HCA。 -切换意味着建立新 QP，重新交换正确的 GID/QPN 等信息。 -本地备用端口不一定能访问同一个对端，探测失败应隔离该对端路径。 - -当前全局 context / PD / MR / lkey 模型不能直接支撑跨 HCA 切换。 -跨 HCA 必须先引入 `RdmaDeviceContext`：每个 context 拥有 PD、事件源及 MR 注册表， -IOBuf 内存块按目标 PD 查找 lkey，不能把主设备的 lkey 用于备用设备。 -预注册备用设备需要容量预算及失败回滚；用户注册内存也需按设备注册和保活。 -不能安全注册到备用 PD 的 buffer 走明确的受控复制路径或 TCP，不能使用无效 lkey。 -同 HCA 不同端口若共享 PD 仍需验证 GID、路由和对端可达性；不默认等同于跨 HCA。 - -## SQ 停顿和 RNR 策略 - -SQ 窗口耗尽保留 EAGAIN。可选 watchdog 只跟踪有 outstanding WR 或待发送且被窗口阻塞的连接。 -使用单调时间记录最后发送完成及 credit 前进，不以 idle 时长判断故障。 -待发送数据、未完成 WR 和 credit 的状态分别记录，避免接收数据持续到达掩盖发送停顿。 -超过无进展阈值后使该连接失败并隔离对端路径；不导致整个端口隔离或进程退出。 -RPC 超时与传输 watchdog 独立，后者不延长任何调用的 deadline。 - -ACK timeout 指数、retry count 和 RNR retry 保持现有值作为默认，分别校验 verbs 范围。 -首版不提供无限 RNR 重试：它会把错误变成无法及时检测的停顿。 -允许的有限 RNR 重试需结合收包 buffer 补充延迟测试；调参不能替代滑动窗口正确性。 -watchdog 默认关闭，硬件和拥塞测试确定阈值后再建议生产开启。 - -## 提议配置与可观测性 - -以下名称尚未实现。配置启动时校验，运行时只允许通过专用 API 更新恢复模式。 - -| 参数 | 初始建议 | 约束 | -| --- | --- | --- | -| rdma_recovery_mode | legacy | legacy / auto / tcp_only；auto 显式启用 | -| rdma_path_cooldown_ms | 5000 | 正数，使用单调时钟 | -| rdma_path_max_cooldown_ms | 60000 | 不小于基础冷却时间 | -| rdma_recovery_probe_timeout_ms | 1000 | 正数；探测调用仍遵守更短的原 deadline | -| rdma_send_stall_timeout_ms | 0 | 0 关闭；启用时必须为正数 | -| rdma_backup_paths | 空 | 启动时配置；仅在设备资源域改造后支持 | - -暴露连接失败原因、RDMA-to-TCP 新建链次数、隔离次数、探测成功/失败、 -端口状态、旧 epoch 活动连接数、无进展发送时长及 SQ/RQ credit。 -指标避免使用无界远端地址作为 bvar 名称；对端细节放入有界诊断视图。 -日志包含 socket generation、device、port、WC status/vendor_err、errno 和策略 epoch， -对相同故障做限频，并保留首次根因，避免被 WR_FLUSH 覆盖。 - -## 并发、回收与停机 - -控制任务只持有弱 socket 标识和带代际的 route lease；执行时重新寻址并校验 generation。 -状态发布使用明确的 acquire/release 配对；事件回调不在持有策略锁时调用 SetFailed。 -防止 SetFailed、复活、探测定时器和端口事件之间形成锁顺序循环。 -故障后先停止准入和 WR 提交，再停止对应 CQ 处理并按现有安全流程 reset/drain QP/CQ， -最后释放发送 buffer、MR、PD 和 context。完成状态未知的 DMA 内存不能提前复用。 -停机取消探测和 watchdog、等待回调退出，再销毁设备资源。 -DEVICE_FATAL 的资源回收必须单独验证；无法证明安全排空时不移除当前 FATAL 保护。 - -## 分阶段交付 - -1. 单设备策略、错误分类、端口事件、冷却/探测、TCP 新连接恢复及受控运行时模式。 - 包含 SocketMap/连接池退休与复用边界、代际隔离和基本指标。 -2. 有界发送无进展检测、有限 RNR 参数校验与故障注入；默认保持现有背压行为。 -3. 多设备资源域、按 PD 注册内存及候选路径选择，完成后开放多端口/跨 HCA 主备。 - -每阶段单独实现和验证；阶段一不宣称支持跨 HCA 或 DEVICE_FATAL 无损恢复。 -本设计 PR 只增加文档；后续实现 PR 才启用功能和增加对应测试。 - -## 验证和验收标准 - -| 层级 | 场景 | 验收条件 | +跨 HCA 引入 `RdmaDeviceContext`，分别持有 context、PD、事件源和 MR 注册表。 +IOBuf 和用户内存按目标 PD 查找 lkey，并管理各设备上的注册与引用生命周期。 +备用设备注册失败时选择 TCP。 + +回收顺序为：停止准入和 WR 提交，停止 CQ 处理，reset/drain QP/CQ， +再释放 buffer、MR、PD 和 context。停机先取消探测与 watchdog，再等待回调退出。 + +## SQ 停顿与 RNR + +正常窗口耗尽保持 EAGAIN 背压。发送 watchdog 跟踪 outstanding WR、 +待发送数据和 credit 的进展；超过阈值后使连接失败并隔离对端路径。 +计时从发送受阻开始，发送完成或 credit 前进时更新。 + +ACK timeout、retry count 和 RNR retry 保持当前默认值，支持配置有限重试次数。 +RNR 重试耗尽作为对端路径故障处理，发送停顿检测独立于 RPC deadline。 + +## 配置与指标 + +| 参数 | 建议默认值 | 用途 | | --- | --- | --- | -| 策略单测，注入时钟 | 冷却、退避、epoch 变化、并发准入 | 冷却零 RDMA 准入；到期最多一个探测；旧结果无效 | -| 连接测试，mock verbs | PORT_ERR 与并发建链、复活竞争 | 受影响连接失败并唤醒等待者；后续连接使用 TCP | -| 错误分类 | RNR、RETRY_EXC、WR_FLUSH、post_send errno | RNR 不隔离本地端口；连锁 flush 不重复计数 | -| 背压测试 | SQ/RQ 为零后 CQ/credit 恢复 | EAGAIN 可恢复，无误降级、忙循环或丢失唤醒 | -| watchdog | idle、持续接收但发送停顿、慢恢复 | idle 不触发；真实发送停顿有界失败；恢复更新进度 | -| RPC 语义 | 部分 post 后失败、服务端已执行 | 不在 transport 中重放；deadline 和重试配额不增加 | -| 复用与开关 | TCP_ONLY、AUTO、旧池连接、持续流量 | 新请求不复用退休 RDMA；TCP 成功不算 RDMA 探测成功 | -| 兼容性 | 开关关闭、新旧对端组合 | legacy 行为及现有握手格式不变 | -| 生命周期 | reset/revive、晚到 CQ 回调、停机 | 无 UAF、重复回收、提前释放 DMA buffer 或停机挂起 | -| 双端 Linux RDMA | 拔链路、恢复链路、接收资源不足 | 后续连接可走 TCP；恢复仅经探测；无重连风暴 | -| 双 HCA Linux RDMA | 主路径故障、不同 PD 的 lkey、注册失败 | 备用 QP 使用匹配 MR；失败可退 TCP；无无效 DMA | - -mock 验证不能代替真机验证。硬件验收记录 RDMA provider、固件、拓扑、 -RPC 成功率及恢复时延分布；使用受控 RoCE/IB 测试环境完成端口故障注入。 -分别测量关闭特性、健康 AUTO、冷却 TCP 三种情况下的吞吐、P99 和 CPU。 -合入实现前需给出恢复预算及性能回归容限；本草案不提供未经测量的性能承诺。 - -## 待评审决策 - -- 首阶段运行时控制是否采用进程级 API，后续是否需要 Channel 级覆盖？ -- 长连接退休的默认排空期限和强制失败入口如何与现有连接池接口对齐？ -- 多 HCA 用户内存注册是否提供显式 registration set，还是通过逐设备查询接口？ -- 发送停顿阈值及允许的有限 RNR 重试是否需要按部署配置档提供建议？ +| rdma_recovery_mode | legacy | 选择恢复模式 | +| rdma_path_cooldown_ms | 5000 | 初始冷却时间 | +| rdma_path_max_cooldown_ms | 60000 | 退避上限 | +| rdma_recovery_probe_timeout_ms | 1000 | 探测预算，受调用 deadline 约束 | +| rdma_send_stall_timeout_ms | 0 | 发送停顿阈值，0 关闭 | +| rdma_backup_paths | 空 | 启动时配置主备路径 | + +指标覆盖端口状态、隔离次数、TCP 降级次数、探测结果、旧 epoch 活动连接数和发送停顿时长。 +日志记录 device、port、socket generation、WC status/vendor_err 和 errno。 + +## 实现顺序与验证 + +1. 实现单设备故障分类、端口隔离、冷却探测、TCP 降级和运行时模式切换。 +2. 增加发送停顿检测、有限 RNR 重试配置和诊断指标。 +3. 拆分设备资源域，支持多端口及跨 HCA 主备。 + +单测覆盖冷却与并发探测、错误分类、窗口恢复、模式切换、连接复用和代际回收。 +双端 Linux RDMA 测试覆盖断链恢复、RNR 和发送停顿；双 HCA 测试验证主备切换与 MR 注册。 +对比 LEGACY、健康 AUTO 和降级 TCP 的恢复时延、吞吐、P99 和 CPU。 From 3846c94d373f71b5ecb1a02a1639166081059e34 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Mon, 26 Oct 2026 02:42:31 -0700 Subject: [PATCH 3/8] Add initial RDMA runtime recovery and TCP fallback --- docs/cn/rdma_runtime_recovery.md | 28 ++++- src/brpc/rdma/rdma_endpoint.cpp | 17 ++- src/brpc/rdma/rdma_endpoint.h | 3 + src/brpc/rdma/rdma_helper.cpp | 12 +- src/brpc/rdma/rdma_helper.h | 3 + src/brpc/rdma/rdma_recovery_policy.cpp | 144 ++++++++++++++++++++++ src/brpc/rdma/rdma_recovery_policy.h | 67 ++++++++++ test/brpc_rdma_recovery_unittest.cpp | 156 +++++++++++++++++++++++ test/brpc_rdma_unittest.cpp | 163 +++++++++++++++++++++++++ 9 files changed, 588 insertions(+), 5 deletions(-) create mode 100644 src/brpc/rdma/rdma_recovery_policy.cpp create mode 100644 src/brpc/rdma/rdma_recovery_policy.h create mode 100644 test/brpc_rdma_recovery_unittest.cpp diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index d9621406c4..058a2088f6 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -19,7 +19,33 @@ under the License. # RDMA 运行期故障恢复与 TCP 降级设计 -设计提案，基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 +基于 LinQuickDev/brpc master 的 `87d498dd62d8b3b3562407cdf38ac0c475d312b3`。 + +## 首版实现 + +`RdmaRecoveryPolicy` 已接入客户端和服务端握手入口，控制新连接的 RDMA 准入： + +- LEGACY 为默认模式，沿用现有行为；TCP_ONLY 让新连接使用 TCP。 +- AUTO 下，客户端收到 RETRY_EXC / RNR_RETRY_EXC 后，按远端地址冷却 5 秒。 +- 当前设备的 PORT_ERR 阻止新 RDMA 建链，端口恢复且冷却结束后重新准入。 +- 对端冷却缓存最多 1024 项，满时淘汰最早到期项；模式切换用 epoch 过滤旧连接错误。 + +通过进程级 API 配置： + +```cpp +#include "brpc/rdma/rdma_recovery_policy.h" + +brpc::rdma::SetRdmaRecoveryCooldownMs(5000); +brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::AUTO); +// 运维降级 +brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::TCP_ONLY); +``` + +当前模式切换作用于新建连接，存量连接继续使用已协商的传输。 +冷却到期后重新开放准入，TCP 连接沿用现有复用与重建机制。 +后续迭代增加存量连接退休、单连接探测、指数退避、发送 watchdog 和多端口主备。 + +以下为完整方案。 ## 当前问题 diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 0dc41c9dd1..33eebaa08c 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -29,6 +29,7 @@ #include "brpc/reloadable_flags.h" #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma_transport.h" #include "brpc/rdma/rdma_handshake.h" @@ -131,6 +132,7 @@ RdmaEndpoint::RdmaEndpoint(Socket* s) : _socket(s) , _state(UNINIT) , _handshake_version(0) + , _recovery_epoch(0) , _resource(nullptr) , _send_cq_events(0) , _recv_cq_events(0) @@ -178,6 +180,7 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; + _recovery_epoch = 0; _outgoing_ece.reset(); _resource = nullptr; _send_cq_events = 0; @@ -212,7 +215,10 @@ void RdmaConnect::StartConnect(const Socket* socket, if (Socket::Address(socket->id(), &s) != 0) { return; } - if (!IsRdmaAvailable()) { + if (!IsRdmaAvailable() || + !GlobalRdmaRecoveryPolicy().AllowNewConnection( + butil::endpoint2str(socket->remote_side()).c_str(), + &rdma_transport->_rdma_ep->_recovery_epoch)) { rdma_transport->_rdma_state = RdmaTransport::RDMA_OFF; rdma_transport->_rdma_ep->_state.store( RdmaEndpoint::FALLBACK_TCP, butil::memory_order_release); @@ -683,7 +689,9 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s } // Negotiate + allocate resources. - bool negotiated = r == RemoteHelloResult::NEGOTIATED; + bool negotiated = r == RemoteHelloResult::NEGOTIATED && + GlobalRdmaRecoveryPolicy().AllowNewConnection( + "", &ep->_recovery_epoch); if (negotiated) { ep->ApplyRemoteHello(remote); ep->_state.store(S_ALLOC_QPCQ, butil::memory_order_relaxed); @@ -1812,6 +1820,11 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { + if (s->CreatedByConnect() && IsRdmaPathFailure(wc[i].status)) { + GlobalRdmaRecoveryPolicy().ReportPeerFailure( + butil::endpoint2str(s->remote_side()).c_str(), + ep->_recovery_epoch); + } PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); s->SetFailed(ERDMA, "RDMA completion error(%d) from %s: %s", diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index eda5e558ef..eff186be50 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -307,6 +307,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // 3 = v3 "RDM3" int _handshake_version; + // Admission epoch used to discard failures from an earlier recovery mode. + uint64_t _recovery_epoch; + // ECE payload to advertise in the next local hello: // Client: the locally queried ECE capabilities (filled // before C_HELLO_SEND); diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index 7e671a85ba..f0ff394a37 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -30,6 +30,7 @@ #include "brpc/rdma/block_pool.h" #include "brpc/rdma/rdma_endpoint.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" namespace butil { @@ -313,8 +314,10 @@ static void OnRdmaAsyncEvent(Socket* m) { } case IBV_EVENT_PORT_ACTIVE: case IBV_EVENT_PORT_ERR: { - // Port up/down will lead these two events. - // The port error is recoverable. + if (event.element.port_num == g_port_num) { + GlobalRdmaRecoveryPolicy().ReportPortState( + event.event_type == IBV_EVENT_PORT_ACTIVE); + } IbvAckAsyncEvent(&event); break; } @@ -699,6 +702,11 @@ bool IsRdmaAvailable() { return g_rdma_available.load(butil::memory_order_acquire); } +bool IsRdmaPathFailure(ibv_wc_status status) { + return status == IBV_WC_RETRY_EXC_ERR || + status == IBV_WC_RNR_RETRY_EXC_ERR; +} + void GlobalDisableRdma() { if (g_rdma_available.exchange(false, butil::memory_order_acquire)) { LOG(FATAL) << "RDMA is disabled due to some unrecoverable problem"; diff --git a/src/brpc/rdma/rdma_helper.h b/src/brpc/rdma/rdma_helper.h index 2383d2d8b4..d3a840cf1d 100644 --- a/src/brpc/rdma/rdma_helper.h +++ b/src/brpc/rdma/rdma_helper.h @@ -89,6 +89,9 @@ bool IsRdmaAvailable(); // Disable RDMA in the remaining lifetime of the process void GlobalDisableRdma(); +// Completion failures that indicate an unusable path or exhausted receive credits. +bool IsRdmaPathFailure(ibv_wc_status status); + // If the given protocol supported by RDMA bool SupportedByRdma(std::string protocol); diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp new file mode 100644 index 0000000000..801605cb90 --- /dev/null +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "brpc/rdma/rdma_recovery_policy.h" +#include "butil/time.h" + +namespace brpc { +namespace rdma { + +RdmaRecoveryPolicy::RdmaRecoveryPolicy(Clock clock, size_t max_peers) + : _clock(clock) + , _max_peers(max_peers) + , _mode(RdmaRecoveryMode::LEGACY) + , _epoch(1) + , _cooldown_us(5000000) + , _port_active(true) + , _port_retry_at(0) { +} + +bool RdmaRecoveryPolicy::SetMode(RdmaRecoveryMode mode) { + if (mode != RdmaRecoveryMode::LEGACY && mode != RdmaRecoveryMode::AUTO && + mode != RdmaRecoveryMode::TCP_ONLY) { + return false; + } + std::lock_guard lock(_mutex); + if (_mode != mode) { + _mode = mode; + ++_epoch; + _peers.clear(); + } + return true; +} + +RdmaRecoveryMode RdmaRecoveryPolicy::GetMode() { + std::lock_guard lock(_mutex); + return _mode; +} + +bool RdmaRecoveryPolicy::SetCooldownMs(int64_t cooldown_ms) { + if (cooldown_ms <= 0 || cooldown_ms > INT32_MAX) { + return false; + } + std::lock_guard lock(_mutex); + _cooldown_us = cooldown_ms * 1000; + return true; +} + +bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, + uint64_t* epoch) { + std::lock_guard lock(_mutex); + if (epoch != nullptr) { + *epoch = _epoch; + } + if (_mode == RdmaRecoveryMode::LEGACY) { + return true; + } + if (_mode == RdmaRecoveryMode::TCP_ONLY) { + return false; + } + const int64_t now = _clock(); + if (!_port_active || now < _port_retry_at) { + return false; + } + auto it = _peers.find(peer); + if (it == _peers.end()) { + return true; + } + if (now < it->second) { + return false; + } + _peers.erase(it); + return true; +} + +void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, + uint64_t epoch) { + std::lock_guard lock(_mutex); + if (_mode != RdmaRecoveryMode::AUTO || epoch != _epoch || + peer.empty() || _max_peers == 0) { + return; + } + const int64_t now = _clock(); + for (auto it = _peers.begin(); it != _peers.end();) { + if (it->second <= now) { + it = _peers.erase(it); + } else { + ++it; + } + } + if (_peers.count(peer) == 0 && _peers.size() == _max_peers) { + auto oldest = _peers.begin(); + for (auto it = _peers.begin(); it != _peers.end(); ++it) { + if (it->second < oldest->second) { + oldest = it; + } + } + _peers.erase(oldest); + } + _peers[peer] = now + _cooldown_us; +} + +void RdmaRecoveryPolicy::ReportPortState(bool active) { + std::lock_guard lock(_mutex); + _port_active = active; + if (!active) { + _port_retry_at = _clock() + _cooldown_us; + } +} + +RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { + // Async event callbacks may outlive ordinary static destructors. + static RdmaRecoveryPolicy* policy = + new RdmaRecoveryPolicy(butil::monotonic_time_us); + return *policy; +} + +bool SetRdmaRecoveryMode(RdmaRecoveryMode mode) { + return GlobalRdmaRecoveryPolicy().SetMode(mode); +} + +RdmaRecoveryMode GetRdmaRecoveryMode() { + return GlobalRdmaRecoveryPolicy().GetMode(); +} + +bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms) { + return GlobalRdmaRecoveryPolicy().SetCooldownMs(cooldown_ms); +} + +} // namespace rdma +} // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h new file mode 100644 index 0000000000..d5f430d6db --- /dev/null +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_RDMA_RECOVERY_POLICY_H +#define BRPC_RDMA_RECOVERY_POLICY_H + +#include +#include +#include +#include +#include + +namespace brpc { +namespace rdma { + +enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; + +// Admission policy for new connections on the process's selected RDMA port. +// An empty peer selects server admission, which only checks the local port. +class RdmaRecoveryPolicy { +public: + typedef int64_t (*Clock)(); + explicit RdmaRecoveryPolicy(Clock clock, size_t max_peers = 1024); + bool SetMode(RdmaRecoveryMode mode); + RdmaRecoveryMode GetMode(); + // Positive milliseconds, at most INT32_MAX. Applies to future failures. + bool SetCooldownMs(int64_t cooldown_ms); + bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); + void ReportPeerFailure(const std::string& peer, uint64_t epoch); + void ReportPortState(bool active); + +private: + std::mutex _mutex; + const Clock _clock; + const size_t _max_peers; + RdmaRecoveryMode _mode; + uint64_t _epoch; + int64_t _cooldown_us; + bool _port_active; + int64_t _port_retry_at; + std::map _peers; +}; + +// Process-lifetime policy, shared by client and server handshake admission. +RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy(); +bool SetRdmaRecoveryMode(RdmaRecoveryMode mode); +RdmaRecoveryMode GetRdmaRecoveryMode(); +bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms); + +} // namespace rdma +} // namespace brpc + +#endif // BRPC_RDMA_RECOVERY_POLICY_H diff --git a/test/brpc_rdma_recovery_unittest.cpp b/test/brpc_rdma_recovery_unittest.cpp new file mode 100644 index 0000000000..8470230e57 --- /dev/null +++ b/test/brpc_rdma_recovery_unittest.cpp @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include "brpc/rdma/rdma_recovery_policy.h" + +namespace brpc { +namespace rdma { +namespace { + +std::atomic now_us(1000000); +int64_t FakeClock() { return now_us.load(); } + +class RdmaRecoveryTest : public testing::Test { +protected: + RdmaRecoveryTest() : policy(FakeClock, 2) {} + void SetUp() override { now_us = 1000000; } + RdmaRecoveryPolicy policy; +}; + +TEST_F(RdmaRecoveryTest, legacy_preserves_admission) { + policy.ReportPortState(false); + uint64_t epoch; + EXPECT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.SetMode(RdmaRecoveryMode::AUTO)); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, peer_failure_only_cools_that_peer) { + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetCooldownMs(10); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("")); + now_us += 9999; + EXPECT_FALSE(policy.AllowNewConnection("a")); + ++now_us; + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, port_requires_active_and_elapsed_cooldown) { + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.SetCooldownMs(10); + policy.ReportPortState(false); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); + policy.ReportPortState(true); + EXPECT_FALSE(policy.AllowNewConnection("a")); + now_us += 10000; + EXPECT_TRUE(policy.AllowNewConnection("a")); + policy.ReportPortState(false); + now_us += 10000; + EXPECT_FALSE(policy.AllowNewConnection("a")); + policy.ReportPortState(true); + EXPECT_TRUE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, tcp_only_and_stale_failure_after_mode_change) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t old_epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &old_epoch)); + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); + policy.SetMode(RdmaRecoveryMode::AUTO); + policy.ReportPeerFailure("a", old_epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection("a", &epoch)); + EXPECT_NE(old_epoch, epoch); + // Reapplying AUTO does not clear existing quarantine. + policy.ReportPeerFailure("a", epoch); + policy.SetMode(RdmaRecoveryMode::AUTO); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, bounded_peer_cache_evicts_earliest_expiry) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + ++now_us; + policy.ReportPeerFailure("b", epoch); + ++now_us; + policy.ReportPeerFailure("c", epoch); + EXPECT_TRUE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_FALSE(policy.AllowNewConnection("c")); + now_us += 5000000; + policy.ReportPeerFailure("d", epoch); + EXPECT_TRUE(policy.AllowNewConnection("b")); + EXPECT_TRUE(policy.AllowNewConnection("c")); + EXPECT_FALSE(policy.AllowNewConnection("d")); +} + +TEST_F(RdmaRecoveryTest, validates_configuration) { + EXPECT_FALSE(policy.SetMode(static_cast(-1))); + EXPECT_EQ(RdmaRecoveryMode::LEGACY, policy.GetMode()); + EXPECT_FALSE(policy.SetCooldownMs(0)); + EXPECT_FALSE(policy.SetCooldownMs(-1)); + EXPECT_FALSE(policy.SetCooldownMs(static_cast(INT32_MAX) + 1)); + EXPECT_TRUE(policy.SetCooldownMs(INT32_MAX)); + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.ReportPeerFailure("a", epoch); + EXPECT_FALSE(policy.AllowNewConnection("a")); +} + +TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { + std::vector workers; + for (int i = 0; i < 8; ++i) { + workers.emplace_back([this, i] { + for (int j = 0; j < 200; ++j) { + uint64_t epoch; + const std::string peer = std::to_string(i); + policy.SetMode(j % 2 ? RdmaRecoveryMode::AUTO : + RdmaRecoveryMode::TCP_ONLY); + policy.AllowNewConnection(peer, &epoch); + policy.ReportPeerFailure(peer, epoch); + } + }); + } + for (auto& worker : workers) { + worker.join(); + } + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("a")); + EXPECT_FALSE(policy.AllowNewConnection("")); +} + +} // namespace +} // namespace rdma +} // namespace brpc diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index 6ec54ed90b..bd808ca0bc 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -35,6 +35,7 @@ #include "brpc/acceptor.h" #include "brpc/channel.h" #include "brpc/controller.h" +#include "brpc/input_messenger.h" #include "brpc/server.h" #include "brpc/socket.h" #include "brpc/errno.pb.h" @@ -47,6 +48,7 @@ #include "brpc/rdma/rdma_handshake_constants.h" #include "brpc/rdma/rdma_handshake.pb.h" #include "brpc/rdma/rdma_helper.h" +#include "brpc/rdma/rdma_recovery_policy.h" #include "echo.pb.h" static const int PORT = 8713; @@ -3316,6 +3318,167 @@ TEST(RdmaInlineDataTest, should_post_inline) { EXPECT_FALSE(rdma::ShouldPostInline(true, 1, 0)); } +class RdmaAdmissionTest : public RdmaTest { +protected: + void SetUp() override { + _saved_mode = rdma::GetRdmaRecoveryMode(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + } + + void TearDown() override { + StopServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(_saved_mode); + RdmaTest::TearDown(); + } + + void CallEchoOverFallback() { + Channel channel; + ChannelOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.connect_timeout_ms = 1000; + options.timeout_ms = 1000; + options.max_retry = 0; + ASSERT_EQ(0, channel.Init(g_ep, &options)); + Controller cntl; + test::EchoRequest req; + test::EchoResponse res; + req.set_message("runtime recovery"); + test::EchoService::Stub(&channel).Echo(&cntl, &req, &res, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ("MyEchoService", res.message()); + SocketUniquePtr s; + ASSERT_EQ(0, Socket::Address(cntl._single_server_id, &s)); + EXPECT_EQ(RdmaTransport::RDMA_OFF, RdmaTransportOf(s)->_rdma_state); + EXPECT_EQ(rdma::RdmaEndpoint::FALLBACK_TCP, + RdmaTransportOf(s)->_rdma_ep->_state); + EXPECT_FALSE(s->Failed()); + } + +private: + rdma::RdmaRecoveryMode _saved_mode; +}; + +TEST_F(RdmaAdmissionTest, tcp_only_carries_rpc_to_tcp_server) { + StartServer(false); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, cooled_peer_carries_rpc_over_tcp) { + StartServer(); + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + policy.ReportPeerFailure(peer, epoch); + ASSERT_FALSE(policy.AllowNewConnection(peer)); + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +static int PollRnrCompletion(ibv_cq* cq, int, ibv_wc* wc) { + bool* delivered = static_cast(cq->cq_context); + if (*delivered) { + return 0; + } + *delivered = true; + memset(wc, 0, sizeof(*wc)); + wc->status = IBV_WC_RNR_RETRY_EXC_ERR; + return 1; +} + +TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { + StartServer(); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + SocketOptions cq_options; + cq_options.user = ep; + SocketId cq_sid; + ASSERT_EQ(0, Socket::Create(cq_options, &cq_sid)); + SocketUniquePtr cq_socket; + ASSERT_EQ(0, Socket::Address(cq_sid, &cq_socket)); + + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + uint64_t epoch; + ASSERT_TRUE(policy.AllowNewConnection(peer, &epoch)); + bool delivered = false; + ibv_context context{}; + context.ops.poll_cq = PollRnrCompletion; + ibv_cq cq{}; + cq.context = &context; + cq.cq_context = &delivered; + rdma::RdmaResource resource; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->_cq_sid = cq_sid; + ep->_recovery_epoch = epoch; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const bool saved_polling = rdma::FLAGS_rdma_use_polling; + rdma::FLAGS_rdma_use_polling = true; + + rdma::RdmaEndpoint::PollCq(cq_socket.get()); + EXPECT_TRUE(delivered); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_TRUE(policy.AllowNewConnection("127.0.0.1:9999")); + + // The test owns the mock CQ and resource; ordinary recycling owns neither. + ep->_resource = nullptr; + ep->_cq_sid = INVALID_SOCKET_ID; + resource.polling_cq = nullptr; + cq_socket->_user = nullptr; + cq_socket->SetFailed(); + rdma::FLAGS_rdma_use_polling = saved_polling; + ASSERT_NO_FATAL_FAILURE(CallEchoOverFallback()); +} + +TEST_F(RdmaAdmissionTest, server_declines_v2_and_v3_when_tcp_only) { + StartServer(); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::TCP_ONLY); + for (int version : {2, 3}) { + butil::fd_guard fd; + ASSERT_NO_FATAL_FAILURE(ConnectToServer(&fd)); + if (version == 2) { + uint8_t hello[rdma::HELLO_V2_MSG_LEN_MIN]; + MakeV2ClientHello(hello); + ASSERT_TRUE(WriteAll(fd, hello, sizeof(hello))); + } else { + const std::string hello = MakeV3Packet(MakeValidV3Hello()); + ASSERT_TRUE(WriteAll(fd, hello.data(), hello.size())); + } + Socket* s = WaitForServerSocket(); + ASSERT_TRUE(s != nullptr); + auto* transport = RdmaTransportOf(s); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::S_ACK_WAIT, transport); + EXPECT_EQ(RdmaTransport::RDMA_OFF, transport->_rdma_state); + EXPECT_EQ(nullptr, transport->_rdma_ep->_resource); + const uint32_t ack = 0; + ASSERT_TRUE(WriteAll(fd, &ack, sizeof(ack))); + ASSERT_RDMA_STATE(rdma::RdmaEndpoint::FALLBACK_TCP, transport); + EXPECT_FALSE(s->Failed()); + fd.reset(-1); + ASSERT_TRUE(WaitForServerSocketGone()); + } +} + +TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RETRY_EXC_ERR)); + EXPECT_TRUE(rdma::IsRdmaPathFailure(IBV_WC_RNR_RETRY_EXC_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_SUCCESS)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_WR_FLUSH_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_LOC_PROT_ERR)); + EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); +} + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { From 15ad2ffb85dc986a77e2feac41c0bee5e63d32dd Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 02:23:38 -0700 Subject: [PATCH 4/8] Unify RDMA failure reporting and add recovery diagnostics --- docs/cn/rdma_runtime_recovery.md | 6 + example/BUILD.bazel | 10 +- example/rdma_performance/CMakeLists.txt | 4 + example/rdma_performance/Makefile | 11 +- example/rdma_performance/RECOVERY.md | 94 ++++++++++++++ example/rdma_performance/recovery_client.cpp | 122 +++++++++++++++++++ src/brpc/rdma/rdma_endpoint.cpp | 93 ++++++++++++-- src/brpc/rdma/rdma_endpoint.h | 8 ++ src/brpc/rdma/rdma_helper.cpp | 8 +- src/brpc/rdma/rdma_recovery_policy.cpp | 101 ++++++++++++++- src/brpc/rdma/rdma_recovery_policy.h | 29 ++++- test/brpc_rdma_recovery_unittest.cpp | 30 +++++ test/brpc_rdma_unittest.cpp | 107 ++++++++++++++++ 13 files changed, 605 insertions(+), 18 deletions(-) create mode 100644 example/rdma_performance/RECOVERY.md create mode 100644 example/rdma_performance/recovery_client.cpp diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index 058a2088f6..acb9a72b6a 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -29,6 +29,12 @@ under the License. - AUTO 下，客户端收到 RETRY_EXC / RNR_RETRY_EXC 后，按远端地址冷却 5 秒。 - 当前设备的 PORT_ERR 阻止新 RDMA 建链，端口恢复且冷却结束后重新准入。 - 对端冷却缓存最多 1024 项，满时淘汰最早到期项；模式切换用 epoch 过滤旧连接错误。 +- 故障上报覆盖 QP/CQ 异步错误、CQ 轮询/事件/通知错误及链路类发送错误，同一连接只计一次。 +- QP/CQ 绑定所属 socket，异步事件核对当前资源；发生运行期错误的资源直接释放。 + +`GetRdmaRecoveryStatus()` 和 `/vars/rdma_recovery_*` 暴露冷却状态、分原因故障计数、 +TCP 回落准入数及 RDMA 协商数。实机步骤见 +[RDMA 双机故障恢复测试](../../example/rdma_performance/RECOVERY.md)。 通过进程级 API 配置： diff --git a/example/BUILD.bazel b/example/BUILD.bazel index 05eda2080b..9049cf07d7 100644 --- a/example/BUILD.bazel +++ b/example/BUILD.bazel @@ -123,6 +123,14 @@ cc_binary( ], ) +cc_binary( + name = "rdma_recovery_client", + srcs = ["rdma_performance/recovery_client.cpp"], + includes = ["rdma_performance"], + copts = COPTS + ["-DBRPC_WITH_RDMA=1"], + deps = [":cc_rdma_performance_proto", "//:brpc"], +) + cc_binary( name = "ubring_performance_server", srcs = [ @@ -162,4 +170,4 @@ cc_binary( deps = [ "//:brpc", ], -) \ No newline at end of file +) diff --git a/example/rdma_performance/CMakeLists.txt b/example/rdma_performance/CMakeLists.txt index 213cff0f24..7e0e70f724 100644 --- a/example/rdma_performance/CMakeLists.txt +++ b/example/rdma_performance/CMakeLists.txt @@ -40,3 +40,7 @@ brpc_example_configure_target(server) target_link_libraries(client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) target_link_libraries(server PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) + +add_executable(recovery_client recovery_client.cpp ${PROTO_SRC} ${PROTO_HEADER}) +brpc_example_configure_target(recovery_client) +target_link_libraries(recovery_client PRIVATE ${BRPC_LIB} ${DYNAMIC_LIB}) diff --git a/example/rdma_performance/Makefile b/example/rdma_performance/Makefile index c112f1c330..58b40acf8b 100644 --- a/example/rdma_performance/Makefile +++ b/example/rdma_performance/Makefile @@ -59,7 +59,7 @@ endif .PHONY:all ifdef WITH_RDMA -all: client server +all: client server recovery_client else all: endif @@ -67,7 +67,7 @@ endif .PHONY:clean clean: @echo "> Cleaning" - rm -rf client server $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) + rm -rf client server recovery_client recovery_client.o $(PROTO_GENS) $(PROTO_OBJS) $(CLIENT_OBJS) $(SERVER_OBJS) client:$(PROTO_OBJS) $(CLIENT_OBJS) @echo "> Linking $@" @@ -85,6 +85,13 @@ else $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ endif +recovery_client:$(PROTO_OBJS) recovery_client.o +ifneq ("$(LINK_SO)", "") + $(CXX) $(LIBPATHS) $(SOPATHS) $(LINK_OPTIONS_SO) -o $@ +else + $(CXX) $(LIBPATHS) $(LINK_OPTIONS) -o $@ +endif + %.pb.cc %.pb.h:%.proto @echo "> Generating $@" $(PROTOC) --cpp_out=. --proto_path=. $(PROTOC_EXTRA_ARGS) $< diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md new file mode 100644 index 0000000000..9b074ce8c1 --- /dev/null +++ b/example/rdma_performance/RECOVERY.md @@ -0,0 +1,94 @@ + + +# RDMA 双机故障恢复测试 + +两台 Linux 主机运行 RDMA 服务端和 recovery_client，TCP 地址通过独立可达的网络通信， +故障注入作用于 RDMA 测试链路。记录设备、端口、GID、驱动和固件版本。 + +## 编译 + +在仓库根目录执行： + +```bash +cmake -S . -B build -DWITH_RDMA=ON -DBUILD_SHARED_LIBS=OFF +cmake --build build -j8 +cmake --install build --prefix "$PWD/build/install" +cmake -S example/rdma_performance -B build/recovery -DCMAKE_PREFIX_PATH="$PWD/build/install" +cmake --build build/recovery --target server recovery_client -j8 +``` + +Bazel 入口为 `//example:rdma_recovery_client`，Make 入口为示例目录中的 `recovery_client`。 + +## 运行与切换 + +主机 B 启动服务端，按环境选择 RDMA 设备及 GID： + +```bash +./build/recovery/server --port=8002 --use_rdma=true --rdma_device=mlx5_0 --rdma_gid_index=3 +``` + +主机 A 启动客户端，将 B_TCP_IP 替换为服务端 TCP 地址： + +```bash +printf 'auto\n' > /tmp/brpc-rdma-mode +./build/recovery/recovery_client --server=B_TCP_IP:8002 --seconds=180 \ + --mode_file=/tmp/brpc-rdma-mode --connection_type=short \ + --rdma_device=mlx5_0 --rdma_gid_index=3 +``` + +客户端持续发送 RPC，失败后继续下一次调用。每秒输出累计成功数、失败数、RDMA 协商数、 +TCP 回落准入数、冷却对端数和端口状态。运行中修改模式文件： + +```bash +printf 'tcp_only\n' > /tmp/brpc-rdma-mode +printf 'auto\n' > /tmp/brpc-rdma-mode +``` + +`short` 为每次调用建立连接，便于验证准入和冷却后的 RDMA 重建。 +`single` 用于观察存量连接：已回落的 TCP 连接继续复用，模式切换影响后续新连接。 + +客户端诊断入口： + +```bash +curl 'http://127.0.0.1:8001/vars/rdma_recovery_*' +``` + +`rdma_recovery_status` 提供模式、epoch、端口状态、冷却剩余时间和分原因故障计数； +`rdma_recovery_tcp_fallback_admissions` 为策略选择 TCP 的累计准入次数。 +计数以连接准入为单位，独立于 RPC 成功数。 + +## 测试步骤 + +| 场景 | 操作 | 观测 | +| --- | --- | --- | +| 健康链路 | AUTO 运行 | RPC 成功，RDMA 协商数持续增加 | +| 运维降级 | 文件改为 tcp_only | 后续连接成功，TCP 准入数增加，RDMA 协商数停止增加 | +| 恢复模式 | 文件改回 auto | short 模式下 RDMA 协商数恢复增加 | +| 端口故障 | 关闭客户端 RDMA 测试端口，保持 TCP 地址可达 | port_active=false，新连接经 TCP 成功 | +| 端口恢复 | 恢复测试端口，等待 cooldown_ms | 端口变为 active，冷却结束后 RDMA 协商数恢复增加 | +| 在途故障 | single 模式下断开 RDMA 链路 | 记录在途失败和恢复时延，后续连接经 TCP 继续调用 | +| 对端故障 | 中断服务端 RDMA 数据路径，保持 TCP 服务 | retry_exhausted / qp_fatal 等计数增加，对端进入冷却 | + +RNR 测试需要接收端故障注入，暂停补充接收 WR 并突破接收 credit 保护后观察 rnr 计数。 +普通拥塞可能只表现为背压；源码 mock CQE 测试可确定性验证 RNR 分支。 + +使用 `--use_rdma=false` 与 TCP 服务端可做无网卡烟测。 +客户端退出码 0 表示至少一次 RPC 成功，2 表示整个测试均失败；故障窗口内的失败数单独查看。 +验收记录故障前后成功率、恢复时延和各原因计数；性能基线使用原 performance client。 diff --git a/example/rdma_performance/recovery_client.cpp b/example/rdma_performance/recovery_client.cpp new file mode 100644 index 0000000000..9b4a36badf --- /dev/null +++ b/example/rdma_performance/recovery_client.cpp @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/rdma/rdma_recovery_policy.h" +#include "butil/time.h" +#include "test.pb.h" + +DEFINE_string(server, "127.0.0.1:8002", "Performance test server address"); +DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); +DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); +DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); +DEFINE_bool(use_rdma, true, "Use RDMA; false runs a TCP smoke test"); +DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); +DEFINE_int32(interval_ms, 100, "Pause between RPCs"); +DEFINE_int32(timeout_ms, 1000, "RPC deadline"); +DEFINE_int32(cooldown_ms, 5000, "Path cooldown"); +DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); + +static bool SetMode(const std::string& value) { + using brpc::rdma::RdmaRecoveryMode; + if (value == "legacy") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::LEGACY); + } + if (value == "auto") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::AUTO); + } + if (value == "tcp_only") { + return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::TCP_ONLY); + } + return false; +} + +int main(int argc, char* argv[]) { + GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || + !brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms) || + !SetMode(FLAGS_mode)) { + std::cerr << "Invalid duration, interval, timeout, cooldown or mode\n"; + return 1; + } + brpc::Channel channel; + brpc::ChannelOptions options; + options.socket_mode = FLAGS_use_rdma ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.connection_type = FLAGS_connection_type; + options.timeout_ms = FLAGS_timeout_ms; + options.connect_timeout_ms = FLAGS_timeout_ms; + options.max_retry = 0; + if (channel.Init(FLAGS_server.c_str(), &options) != 0) { + return 1; + } + if (FLAGS_dummy_port && brpc::StartDummyServerAt(FLAGS_dummy_port) != 0) { + return 1; + } + std::string mode = FLAGS_mode; + uint64_t succeeded = 0; + uint64_t failed = 0; + const int64_t start = butil::monotonic_time_us(); + int64_t next_report = start; + std::cout << "seconds,mode,success,failure,rdma_negotiated,tcp_fallback_admissions," + "cooled_peers,port_active,port_cooldown_ms\n"; + while (!brpc::IsAskedToQuit() && + butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { + if (!FLAGS_mode_file.empty()) { + std::ifstream input(FLAGS_mode_file); + std::string value; + if (input >> value && value != mode) { + if (SetMode(value)) { + mode = value; + } else { + std::cerr << "Ignoring invalid recovery mode: " << value << '\n'; + } + } + } + test::PerfTestRequest request; + request.set_echo_attachment(false); + test::PerfTestResponse response; + brpc::Controller cntl; + test::PerfTestService_Stub(&channel).Test(&cntl, &request, &response, nullptr); + if (cntl.Failed()) { + ++failed; + std::cerr << "RPC failure: " << cntl.ErrorText() << '\n'; + } else { + ++succeeded; + } + const int64_t now = butil::monotonic_time_us(); + if (now >= next_report) { + const auto s = brpc::rdma::GetRdmaRecoveryStatus(); + std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded + << ',' << failed << ',' << s.negotiated_connections << ',' + << s.tcp_only_admissions + s.port_cooldown_admissions + + s.peer_cooldown_admissions + << ',' << s.cooled_peers << ',' << s.port_active << ',' + << s.port_cooldown_ms << std::endl; + next_report = now + 1000000; + } + usleep(static_cast(FLAGS_interval_ms) * 1000); + } + std::cout << "completed success=" << succeeded << " failure=" << failed << std::endl; + return succeeded ? 0 : 2; +} diff --git a/src/brpc/rdma/rdma_endpoint.cpp b/src/brpc/rdma/rdma_endpoint.cpp index 33eebaa08c..3d78e47a75 100644 --- a/src/brpc/rdma/rdma_endpoint.cpp +++ b/src/brpc/rdma/rdma_endpoint.cpp @@ -133,6 +133,7 @@ RdmaEndpoint::RdmaEndpoint(Socket* s) , _state(UNINIT) , _handshake_version(0) , _recovery_epoch(0) + , _runtime_failure(RdmaFailureReason::NONE) , _resource(nullptr) , _send_cq_events(0) , _recv_cq_events(0) @@ -181,6 +182,7 @@ void RdmaEndpoint::Reset() { _state.store(UNINIT, butil::memory_order_relaxed); _handshake_version = 0; _recovery_epoch = 0; + _runtime_failure.store(RdmaFailureReason::NONE, butil::memory_order_relaxed); _outgoing_ece.reset(); _resource = nullptr; _send_cq_events = 0; @@ -242,6 +244,50 @@ void RdmaConnect::StartConnect(const Socket* socket, void RdmaConnect::StopConnect(Socket* socket) { } +void RdmaEndpoint::ReportFailure(RdmaFailureReason reason) { + RdmaFailureReason expected = RdmaFailureReason::NONE; + if (reason == RdmaFailureReason::NONE || + !_runtime_failure.compare_exchange_strong(expected, reason)) { + return; + } + if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { + return; + } + GlobalRdmaRecoveryPolicy().ReportPeerFailure( + _socket->CreatedByConnect() ? + butil::endpoint2str(_socket->remote_side()).c_str() : "", + _recovery_epoch, reason); + LOG(WARNING) << "RDMA runtime failure=" << RdmaFailureReasonName(reason) + << " socket=" << _socket->id() + << " port=" << static_cast(GetRdmaPortNum()) + << " epoch=" << _recovery_epoch; +} + +void RdmaEndpoint::OnAsyncError(Socket* socket, const ibv_async_event& event) { + auto* transport = dynamic_cast(socket->_transport.get()); + RdmaEndpoint* ep = transport ? transport->GetRdmaEp() : nullptr; + if (!ep || !ep->_resource) { + return; + } + RdmaFailureReason reason; + if (event.event_type == IBV_EVENT_CQ_ERR) { + if (event.element.cq != ep->_resource->send_cq && + event.element.cq != ep->_resource->recv_cq && + event.element.cq != ep->_resource->polling_cq) { + return; + } + reason = RdmaFailureReason::CQ_ERROR; + } else { + if (event.element.qp != ep->_resource->qp) { + return; + } + reason = event.event_type == IBV_EVENT_QP_FATAL ? + RdmaFailureReason::QP_FATAL : RdmaFailureReason::ACCESS_ERROR; + } + ep->ReportFailure(reason); + socket->SetFailed(ERDMA, "RDMA async error: %s", RdmaFailureReasonName(reason)); +} + void RdmaConnect::Run() { _done(errno, _data); } @@ -587,6 +633,7 @@ void* RdmaEndpoint::ProcessHandshakeAtClient(void* arg) { if (rdma_transport->_rdma_state == RdmaTransport::RDMA_ON) { ep->_state.store(ESTABLISHED, butil::memory_order_release); + GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); // The handshake is over, so the QP stream may start parsing now. if (ep->StartCqEvents() < 0) { const int saved_errno = errno; @@ -773,6 +820,7 @@ ParseResult RdmaEndpoint::ExecuteServerHandshake(butil::IOBuf* source, Socket* s << ") on " << s->description(); rdma_transport->_rdma_state = RdmaTransport::RDMA_ON; ep->_state.store(ESTABLISHED, butil::memory_order_release); + GlobalRdmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); s->reset_parsing_context(nullptr); // Two things are deliberately not done here. @@ -985,6 +1033,9 @@ ssize_t RdmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { std::ostringstream oss; DebugInfo(oss, ", "); LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); + if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { + ReportFailure(RdmaFailureReason::SEND_POST); + } errno = err; return -1; } @@ -1035,6 +1086,10 @@ int RdmaEndpoint::SendImm(uint32_t imm) { // We use other way to guarantee the Send Queue is not full. // So we just consider this error as an unrecoverable error. LOG(WARNING) << "Fail to ibv_post_send: " << berror(err) << " " << oss.str(); + if (err == EIO || err == ENETDOWN || err == ENETUNREACH || err == ETIMEDOUT) { + ReportFailure(RdmaFailureReason::SEND_POST); + } + errno = err; return -1; } @@ -1052,8 +1107,7 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (0 == wc.wr_id) { _sq_imm_window_size += 1; // If there are any unacknowledged recvs, send an ack. - SendAck(0); - return 0; + return SendAck(0); } // Update SQ window. uint16_t wnd_to_update = wc.wr_id; @@ -1106,8 +1160,8 @@ ssize_t RdmaEndpoint::HandleCompletion(ibv_wc& wc) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (wc.byte_len > 0) { - SendAck(1); + if (wc.byte_len > 0 && SendAck(1) < 0) { + return -1; } return wc.byte_len; } @@ -1342,6 +1396,8 @@ int RdmaEndpoint::DoAllocateResources() { return -1; } + BindResourceToSocket(); + if (!FLAGS_rdma_use_polling) { if (0 != ReqNotifyCq(true, false)) { return -1; @@ -1367,6 +1423,20 @@ int RdmaEndpoint::DoAllocateResources() { return 0; } +void RdmaEndpoint::BindResourceToSocket() { + void* owner = reinterpret_cast(_socket->id()); + _resource->qp->qp_context = owner; + if (_resource->send_cq) { + _resource->send_cq->cq_context = owner; + } + if (_resource->recv_cq) { + _resource->recv_cq->cq_context = owner; + } + if (_resource->polling_cq) { + _resource->polling_cq->cq_context = owner; + } +} + int RdmaEndpoint::StartCqEvents() { if (InputMessengerProcessor::STREAM_NONE != _socket->parsing_stream_type()) { LOG(WARNING) << "StartCqEvents() called while " << *_socket << " is parsing"; @@ -1552,7 +1622,8 @@ void RdmaEndpoint::DeallocateResources() { PollerRemoveCqSid(); } bool move_to_rdma_resource_list = false; - if (_sq_size <= FLAGS_rdma_prepared_qp_size && + if (_runtime_failure.load(butil::memory_order_relaxed) == RdmaFailureReason::NONE && + _sq_size <= FLAGS_rdma_prepared_qp_size && _rq_size <= FLAGS_rdma_prepared_qp_size && FLAGS_rdma_prepared_qp_cnt > 0) { ibv_qp_attr attr; @@ -1659,6 +1730,7 @@ int RdmaEndpoint::GetAndAckEvents(SocketUniquePtr& s) { if (errno != EAGAIN) { const int saved_errno = errno; PLOG(ERROR) << "Fail to get cq event from " << s->description(); + ReportFailure(RdmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to get cq event from %s: %s", s->description().c_str(), berror(saved_errno)); return -1; @@ -1699,6 +1771,7 @@ int RdmaEndpoint::ReqNotifyCq(bool send_cq, bool fatal_on_error) { PLOG(WARNING) << "Fail to arm " << (send_cq ? "send" : "recv") << " CQ comp channel from " << _socket->description(); if (fatal_on_error) { + ReportFailure(RdmaFailureReason::CQ_NOTIFY); _socket->SetFailed(err, "Fail to arm %s CQ channel from %s: %s", send_cq ? "send" : "recv", _socket->description().c_str(), berror(err)); @@ -1752,6 +1825,7 @@ void RdmaEndpoint::PollCq(Socket* m) { if (cnt < 0) { const int saved_errno = errno; PLOG(WARNING) << "Fail to poll cq: " << s->description(); + ep->ReportFailure(RdmaFailureReason::CQ_POLL); s->SetFailed(saved_errno, "Fail to poll cq from %s: %s", s->description().c_str(), berror(saved_errno)); return; @@ -1820,10 +1894,11 @@ void RdmaEndpoint::PollCq(Socket* m) { } if (wc[i].status != IBV_WC_SUCCESS) { - if (s->CreatedByConnect() && IsRdmaPathFailure(wc[i].status)) { - GlobalRdmaRecoveryPolicy().ReportPeerFailure( - butil::endpoint2str(s->remote_side()).c_str(), - ep->_recovery_epoch); + if (IsRdmaPathFailure(wc[i].status)) { + ep->ReportFailure(wc[i].status == IBV_WC_RNR_RETRY_EXC_ERR ? + RdmaFailureReason::RNR : RdmaFailureReason::RETRY_EXHAUSTED); + } else if (wc[i].status != IBV_WC_WR_FLUSH_ERR) { + ep->ReportFailure(RdmaFailureReason::ACCESS_ERROR); } PLOG(WARNING) << "Fail to handle RDMA completion, error status(" << wc[i].status << "): " << s->description(); diff --git a/src/brpc/rdma/rdma_endpoint.h b/src/brpc/rdma/rdma_endpoint.h index eff186be50..85d3b280c9 100644 --- a/src/brpc/rdma/rdma_endpoint.h +++ b/src/brpc/rdma/rdma_endpoint.h @@ -32,6 +32,7 @@ #include "butil/containers/optional.h" #include "brpc/socket.h" #include "brpc/rdma/rdma_handshake_server.h" +#include "brpc/rdma/rdma_recovery_policy.h" namespace brpc { @@ -136,6 +137,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // Real handshake for RDMA-mode sockets. static ParseResult ExecuteServerHandshake(butil::IOBuf* source, Socket* socket); + // Called with an addressed Socket held until its async event is acknowledged. + static void OnAsyncError(Socket* socket, const ibv_async_event& event); + // Initialize polling mode static int PollingModeInitialize(bthread_tag_t tag, std::function callback, @@ -145,6 +149,9 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); static void PollingModeRelease(bthread_tag_t tag); private: + void ReportFailure(RdmaFailureReason reason); + void BindResourceToSocket(); + enum State { UNINIT = 0x0, C_ALLOC_QPCQ = 0x1, @@ -309,6 +316,7 @@ friend int v3_wire::WriteV3Hello(RdmaEndpoint*, const RdmaHello&); // Admission epoch used to discard failures from an earlier recovery mode. uint64_t _recovery_epoch; + butil::atomic _runtime_failure; // ECE payload to advertise in the next local hello: // Client: the locally queried ECE capabilities (filled diff --git a/src/brpc/rdma/rdma_helper.cpp b/src/brpc/rdma/rdma_helper.cpp index f0ff394a37..1ff7884acd 100644 --- a/src/brpc/rdma/rdma_helper.cpp +++ b/src/brpc/rdma/rdma_helper.cpp @@ -269,7 +269,7 @@ static void OnRdmaAsyncEvent(Socket* m) { SocketId sid = (SocketId)event.element.qp->qp_context; SocketUniquePtr s; if (Socket::Address(sid, &s) == 0) { - s->SetFailed(ERDMA, "QP fatal error"); + RdmaEndpoint::OnAsyncError(s.get(), event); LOG(WARNING) << "Receive a QP fatal error on " << s->description(); } // NOTE: @@ -281,7 +281,11 @@ static void OnRdmaAsyncEvent(Socket* m) { break; } case IBV_EVENT_CQ_ERR: { - LOG(WARNING) << "CQ overruns, the connection will be stopped."; + SocketUniquePtr s; + const SocketId sid = reinterpret_cast(event.element.cq->cq_context); + if (Socket::Address(sid, &s) == 0) { + RdmaEndpoint::OnAsyncError(s.get(), event); + } IbvAckAsyncEvent(&event); break; } diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp index 801605cb90..fbaa4d6fd2 100644 --- a/src/brpc/rdma/rdma_recovery_policy.cpp +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -17,10 +17,27 @@ #include "brpc/rdma/rdma_recovery_policy.h" #include "butil/time.h" +#include "bvar/passive_status.h" namespace brpc { namespace rdma { +const char* RdmaFailureReasonName(RdmaFailureReason reason) { + switch (reason) { + case RdmaFailureReason::NONE: return "none"; + case RdmaFailureReason::RETRY_EXHAUSTED: return "retry_exhausted"; + case RdmaFailureReason::RNR: return "rnr"; + case RdmaFailureReason::QP_FATAL: return "qp_fatal"; + case RdmaFailureReason::CQ_ERROR: return "cq_error"; + case RdmaFailureReason::CQ_POLL: return "cq_poll"; + case RdmaFailureReason::CQ_EVENT: return "cq_event"; + case RdmaFailureReason::CQ_NOTIFY: return "cq_notify"; + case RdmaFailureReason::SEND_POST: return "send_post"; + case RdmaFailureReason::ACCESS_ERROR: return "access_error"; + default: return "unknown"; + } +} + RdmaRecoveryPolicy::RdmaRecoveryPolicy(Clock clock, size_t max_peers) : _clock(clock) , _max_peers(max_peers) @@ -69,10 +86,12 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, return true; } if (_mode == RdmaRecoveryMode::TCP_ONLY) { + ++_status.tcp_only_admissions; return false; } const int64_t now = _clock(); if (!_port_active || now < _port_retry_at) { + ++_status.port_cooldown_admissions; return false; } auto it = _peers.find(peer); @@ -80,6 +99,7 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, return true; } if (now < it->second) { + ++_status.peer_cooldown_admissions; return false; } _peers.erase(it); @@ -87,10 +107,16 @@ bool RdmaRecoveryPolicy::AllowNewConnection(const std::string& peer, } void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, - uint64_t epoch) { + uint64_t epoch, + RdmaFailureReason reason) { std::lock_guard lock(_mutex); - if (_mode != RdmaRecoveryMode::AUTO || epoch != _epoch || - peer.empty() || _max_peers == 0) { + if (epoch != _epoch || reason <= RdmaFailureReason::NONE || + reason >= RdmaFailureReason::COUNT) { + return; + } + ++_status.failures[static_cast(reason)]; + if (_mode != RdmaRecoveryMode::AUTO || peer.empty() || _max_peers == 0 || + reason == RdmaFailureReason::ACCESS_ERROR) { return; } const int64_t now = _clock(); @@ -109,6 +135,7 @@ void RdmaRecoveryPolicy::ReportPeerFailure(const std::string& peer, } } _peers.erase(oldest); + ++_status.cache_evictions; } _peers[peer] = now + _cooldown_us; } @@ -117,14 +144,78 @@ void RdmaRecoveryPolicy::ReportPortState(bool active) { std::lock_guard lock(_mutex); _port_active = active; if (!active) { + ++_status.port_errors; _port_retry_at = _clock() + _cooldown_us; } } +RdmaRecoveryStatus RdmaRecoveryPolicy::GetStatus() { + std::lock_guard lock(_mutex); + RdmaRecoveryStatus result = _status; + result.mode = _mode; + result.epoch = _epoch; + result.port_active = _port_active; + const int64_t now = _clock(); + result.port_cooldown_ms = _port_retry_at > now ? + (_port_retry_at - now + 999) / 1000 : 0; + for (const auto& peer : _peers) { + result.cooled_peers += peer.second > now; + } + return result; +} + +void RdmaRecoveryPolicy::RecordNegotiatedConnection(uint64_t epoch) { + std::lock_guard lock(_mutex); + if (epoch == _epoch) { + ++_status.negotiated_connections; + } +} + +static void DescribeRecovery(std::ostream& os, void* arg) { + const auto s = static_cast(arg)->GetStatus(); + os << "{\"mode\":\"" + << (s.mode == RdmaRecoveryMode::AUTO ? "auto" : + s.mode == RdmaRecoveryMode::TCP_ONLY ? "tcp_only" : "legacy") + << "\",\"epoch\":" << s.epoch + << ",\"port_active\":" << (s.port_active ? "true" : "false") + << ",\"port_cooldown_ms\":" << s.port_cooldown_ms + << ",\"cooled_peers\":" << s.cooled_peers + << ",\"tcp_only_admissions\":" << s.tcp_only_admissions + << ",\"port_cooldown_admissions\":" << s.port_cooldown_admissions + << ",\"peer_cooldown_admissions\":" << s.peer_cooldown_admissions + << ",\"port_errors\":" << s.port_errors + << ",\"cache_evictions\":" << s.cache_evictions + << ",\"negotiated_connections\":" << s.negotiated_connections + << ",\"failures\":{"; + for (size_t i = 1; i < static_cast(RdmaFailureReason::COUNT); ++i) { + if (i > 1) { + os << ','; + } + os << '"' << RdmaFailureReasonName(static_cast(i)) + << "\":" << s.failures[i]; + } + os << "}}"; +} + +struct RecoveryMetrics { + explicit RecoveryMetrics(RdmaRecoveryPolicy* policy) + : status("rdma_recovery_status", DescribeRecovery, policy) + , fallbacks("rdma_recovery_tcp_fallback_admissions", [](void* arg) { + const auto s = static_cast(arg)->GetStatus(); + return s.tcp_only_admissions + s.port_cooldown_admissions + + s.peer_cooldown_admissions; + }, policy) { + } + bvar::PassiveStatus status; + bvar::PassiveStatus fallbacks; +}; + RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { // Async event callbacks may outlive ordinary static destructors. static RdmaRecoveryPolicy* policy = new RdmaRecoveryPolicy(butil::monotonic_time_us); + static RecoveryMetrics* metrics = new RecoveryMetrics(policy); + (void)metrics; return *policy; } @@ -140,5 +231,9 @@ bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms) { return GlobalRdmaRecoveryPolicy().SetCooldownMs(cooldown_ms); } +RdmaRecoveryStatus GetRdmaRecoveryStatus() { + return GlobalRdmaRecoveryPolicy().GetStatus(); +} + } // namespace rdma } // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h index d5f430d6db..651cf8ccb5 100644 --- a/src/brpc/rdma/rdma_recovery_policy.h +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -29,6 +29,28 @@ namespace rdma { enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; +enum class RdmaFailureReason { + NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, + CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, COUNT +}; + +const char* RdmaFailureReasonName(RdmaFailureReason reason); + +struct RdmaRecoveryStatus { + RdmaRecoveryMode mode = RdmaRecoveryMode::LEGACY; + uint64_t epoch = 0; + bool port_active = true; + int64_t port_cooldown_ms = 0; + size_t cooled_peers = 0; + uint64_t failures[static_cast(RdmaFailureReason::COUNT)] = {}; + uint64_t tcp_only_admissions = 0; + uint64_t port_cooldown_admissions = 0; + uint64_t peer_cooldown_admissions = 0; + uint64_t port_errors = 0; + uint64_t cache_evictions = 0; + uint64_t negotiated_connections = 0; +}; + // Admission policy for new connections on the process's selected RDMA port. // An empty peer selects server admission, which only checks the local port. class RdmaRecoveryPolicy { @@ -40,8 +62,11 @@ class RdmaRecoveryPolicy { // Positive milliseconds, at most INT32_MAX. Applies to future failures. bool SetCooldownMs(int64_t cooldown_ms); bool AllowNewConnection(const std::string& peer, uint64_t* epoch = nullptr); - void ReportPeerFailure(const std::string& peer, uint64_t epoch); + void ReportPeerFailure(const std::string& peer, uint64_t epoch, + RdmaFailureReason reason = RdmaFailureReason::RETRY_EXHAUSTED); void ReportPortState(bool active); + RdmaRecoveryStatus GetStatus(); + void RecordNegotiatedConnection(uint64_t epoch); private: std::mutex _mutex; @@ -53,6 +78,7 @@ class RdmaRecoveryPolicy { bool _port_active; int64_t _port_retry_at; std::map _peers; + RdmaRecoveryStatus _status; }; // Process-lifetime policy, shared by client and server handshake admission. @@ -60,6 +86,7 @@ RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy(); bool SetRdmaRecoveryMode(RdmaRecoveryMode mode); RdmaRecoveryMode GetRdmaRecoveryMode(); bool SetRdmaRecoveryCooldownMs(int64_t cooldown_ms); +RdmaRecoveryStatus GetRdmaRecoveryStatus(); } // namespace rdma } // namespace brpc diff --git a/test/brpc_rdma_recovery_unittest.cpp b/test/brpc_rdma_recovery_unittest.cpp index 8470230e57..734fe794e2 100644 --- a/test/brpc_rdma_recovery_unittest.cpp +++ b/test/brpc_rdma_recovery_unittest.cpp @@ -151,6 +151,36 @@ TEST_F(RdmaRecoveryTest, concurrent_admission_failure_and_mode_change) { EXPECT_FALSE(policy.AllowNewConnection("")); } +TEST_F(RdmaRecoveryTest, reports_reasons_and_admission_counters) { + policy.SetMode(RdmaRecoveryMode::AUTO); + uint64_t epoch; + policy.AllowNewConnection("a", &epoch); + policy.RecordNegotiatedConnection(epoch); + policy.ReportPeerFailure("a", epoch, RdmaFailureReason::QP_FATAL); + EXPECT_FALSE(policy.AllowNewConnection("a")); + auto status = policy.GetStatus(); + EXPECT_EQ(1u, status.cooled_peers); + EXPECT_EQ(1u, status.negotiated_connections); + EXPECT_EQ(1u, status.peer_cooldown_admissions); + EXPECT_EQ(1u, status.failures[static_cast(RdmaFailureReason::QP_FATAL)]); + policy.ReportPeerFailure("b", epoch, RdmaFailureReason::ACCESS_ERROR); + EXPECT_TRUE(policy.AllowNewConnection("b")); + policy.ReportPortState(false); + EXPECT_FALSE(policy.AllowNewConnection("b")); + status = policy.GetStatus(); + EXPECT_FALSE(status.port_active); + EXPECT_EQ(5000, status.port_cooldown_ms); + EXPECT_EQ(1u, status.port_errors); + EXPECT_EQ(1u, status.port_cooldown_admissions); + now_us += 5000000; + EXPECT_EQ(0u, policy.GetStatus().cooled_peers); + policy.SetMode(RdmaRecoveryMode::TCP_ONLY); + EXPECT_FALSE(policy.AllowNewConnection("b")); + EXPECT_EQ(1u, policy.GetStatus().tcp_only_admissions); + policy.ReportPeerFailure("b", epoch, RdmaFailureReason::CQ_ERROR); + EXPECT_EQ(0u, policy.GetStatus().failures[static_cast(RdmaFailureReason::CQ_ERROR)]); +} + } // namespace } // namespace rdma } // namespace brpc diff --git a/test/brpc_rdma_unittest.cpp b/test/brpc_rdma_unittest.cpp index bd808ca0bc..f6fe8eeff1 100644 --- a/test/brpc_rdma_unittest.cpp +++ b/test/brpc_rdma_unittest.cpp @@ -3426,6 +3426,11 @@ TEST_F(RdmaAdmissionTest, rnr_completion_cools_peer_before_tcp_rpc) { rdma::FLAGS_rdma_use_polling = true; rdma::RdmaEndpoint::PollCq(cq_socket.get()); + const auto first_report = rdma::GetRdmaRecoveryStatus(); + ep->ReportFailure(rdma::RdmaFailureReason::QP_FATAL); + EXPECT_EQ(first_report.failures[static_cast(rdma::RdmaFailureReason::QP_FATAL)], + rdma::GetRdmaRecoveryStatus().failures[ + static_cast(rdma::RdmaFailureReason::QP_FATAL)]); EXPECT_TRUE(delivered); EXPECT_TRUE(socket->Failed()); EXPECT_FALSE(policy.AllowNewConnection(peer)); @@ -3479,6 +3484,108 @@ TEST(RdmaCompletionRecoveryTest, classifies_path_failures) { EXPECT_FALSE(rdma::IsRdmaPathFailure(IBV_WC_REM_ACCESS_ERR)); } +TEST_F(RdmaAdmissionTest, async_errors_validate_owner_and_report_once) { + for (ibv_event_type type : {IBV_EVENT_QP_FATAL, IBV_EVENT_CQ_ERR, + IBV_EVENT_QP_ACCESS_ERR}) { + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::AUTO); + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_qp qp{}; + ibv_qp other_qp{}; + ibv_cq cq{}; + ibv_cq other_cq{}; + rdma::RdmaResource resource; + resource.qp = &qp; + resource.polling_cq = &cq; + ep->_resource = &resource; + ep->BindResourceToSocket(); + EXPECT_EQ(sid, reinterpret_cast(qp.qp_context)); + EXPECT_EQ(sid, reinterpret_cast(cq.cq_context)); + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + const auto before = policy.GetStatus(); + ibv_async_event event{}; + event.event_type = type; + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &other_cq; + } else { + event.element.qp = &other_qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_FALSE(socket->Failed()); + if (type == IBV_EVENT_CQ_ERR) { + event.element.cq = &cq; + } else { + event.element.qp = &qp; + } + rdma::RdmaEndpoint::OnAsyncError(socket.get(), event); + EXPECT_TRUE(socket->Failed()); + const auto reason = type == IBV_EVENT_CQ_ERR ? rdma::RdmaFailureReason::CQ_ERROR : + type == IBV_EVENT_QP_FATAL ? rdma::RdmaFailureReason::QP_FATAL : + rdma::RdmaFailureReason::ACCESS_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + policy.GetStatus().failures[static_cast(reason)]); + EXPECT_EQ(type == IBV_EVENT_QP_ACCESS_ERR, policy.AllowNewConnection(peer)); + ep->ReportFailure(rdma::RdmaFailureReason::RNR); + EXPECT_EQ(before.failures[static_cast(rdma::RdmaFailureReason::RNR)], + policy.GetStatus().failures[static_cast(rdma::RdmaFailureReason::RNR)]); + const std::string exposed = bvar::Variable::describe_exposed("rdma_recovery_status"); + EXPECT_NE(std::string::npos, exposed.find("failures")); + EXPECT_NE(std::string::npos, exposed.find("qp_fatal")); + ep->_resource = nullptr; + resource.qp = nullptr; + resource.polling_cq = nullptr; + } +} + +static int FailAckPost(ibv_qp*, ibv_send_wr*, ibv_send_wr**) { + return EIO; +} + +TEST_F(RdmaAdmissionTest, ack_post_error_propagates_and_cools_peer) { + SocketOptions options; + options.socket_mode = SOCKET_MODE_RDMA; + options.remote_side = g_ep; + options.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(options, &sid)); + SocketUniquePtr socket; + ASSERT_EQ(0, Socket::Address(sid, &socket)); + auto* ep = RdmaTransportOf(socket)->_rdma_ep; + auto& policy = rdma::GlobalRdmaRecoveryPolicy(); + const std::string peer = butil::endpoint2str(g_ep).c_str(); + ASSERT_TRUE(policy.AllowNewConnection(peer, &ep->_recovery_epoch)); + ibv_context context{}; + context.ops.post_send = FailAckPost; + ibv_qp qp{}; + qp.context = &context; + rdma::RdmaResource resource; + resource.qp = &qp; + ep->_resource = &resource; + ep->_state.store(rdma::RdmaEndpoint::ESTABLISHED); + ep->_remote_window_capacity = 0; + ep->_new_rq_wrs.store(1); + ibv_wc wc{}; + wc.opcode = IBV_WC_SEND; + EXPECT_EQ(-1, ep->HandleCompletion(wc)); + EXPECT_EQ(EIO, errno); + EXPECT_FALSE(policy.AllowNewConnection(peer)); + EXPECT_EQ(rdma::RdmaFailureReason::SEND_POST, ep->_runtime_failure.load()); + ep->_resource = nullptr; + resource.qp = nullptr; + socket->SetFailed(); +} + #endif // if BRPC_WITH_RDMA int main(int argc, char* argv[]) { From ba5e1e178121d999356a0058808bba0da59f9f68 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:53:04 -0700 Subject: [PATCH 5/8] Adapt runtime recovery and TCP fallback to URMA --- docs/cn/rdma_runtime_recovery.md | 6 + docs/cn/urma.md | 25 ++ docs/en/urma.md | 29 ++ example/rdma_performance/RECOVERY.md | 8 + example/rdma_performance/recovery_client.cpp | 41 ++- src/brpc/rdma/rdma_recovery_policy.cpp | 42 ++- src/brpc/rdma/rdma_recovery_policy.h | 2 +- src/brpc/urma/urma_endpoint.cpp | 69 +++- src/brpc/urma/urma_endpoint.h | 7 + src/brpc/urma/urma_handshake.cpp | 7 +- src/brpc/urma/urma_recovery_policy.h | 41 +++ test/brpc_urma_unittest.cpp | 336 +++++++++++++++++++ 12 files changed, 590 insertions(+), 23 deletions(-) create mode 100644 src/brpc/urma/urma_recovery_policy.h diff --git a/docs/cn/rdma_runtime_recovery.md b/docs/cn/rdma_runtime_recovery.md index acb9a72b6a..d2141c89a5 100644 --- a/docs/cn/rdma_runtime_recovery.md +++ b/docs/cn/rdma_runtime_recovery.md @@ -51,6 +51,12 @@ brpc::rdma::SetRdmaRecoveryMode(brpc::rdma::RdmaRecoveryMode::TCP_ONLY); 冷却到期后重新开放准入，TCP 连接沿用现有复用与重建机制。 后续迭代增加存量连接退休、单连接探测、指数退避、发送 watchdog 和多端口主备。 +## URMA 适配 + +URMA 复用准入算法，独立维护模式、epoch、对端缓存和 `urma_recovery_*` 指标。 +接入完成错误、JFC 轮询/事件/通知错误及 WR 提交失败，支持 v2/v3 服务端协商降级。 +配置与行为见 [URMA 运行期故障恢复](urma.md#运行期故障恢复)。 + 以下为完整方案。 ## 当前问题 diff --git a/docs/cn/urma.md b/docs/cn/urma.md index 62e10389a8..c207e7d7a1 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -193,3 +193,28 @@ UrmaTransport 推荐用于**大包和跨节点**高吞吐路径，而 UBRing `ContextInitOrDie` 阶段拒绝。 - 硬件数据通路需要受支持的 UMDK provider 和 `liburma`。 - 当前实现面向 Linux。 + +## 运行期故障恢复 + +通过进程级 API 控制新连接的传输选择： + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +默认 LEGACY 沿用现有行为；AUTO 在重试耗尽、RNR、JFC 错误或 WR 提交失败后， +将客户端对端地址冷却 5 秒，期间新连接使用 TCP。TCP_ONLY 让新连接直接选择 TCP。 +服务端通过 v2/v3 hello 拒绝 URMA 协商，随后处理 TCP 请求。 + +URMA 和 RDMA 分别维护模式、epoch、冷却缓存和指标。同一连接只上报首个运行期故障； +模式切换过滤旧 epoch 的错误。访问错误单独计数，WR flush 不重复归因， +SQ/接收 credit 耗尽保留 EAGAIN 背压，ACK 提交失败向连接错误处理传播。 + +`GetUrmaRecoveryStatus()` 和 `/vars/urma_recovery_status` 提供故障原因、冷却状态和协商数， +`/vars/urma_recovery_tcp_fallback_admissions` 提供 TCP 准入次数。 +模式切换作用于新连接；存量连接使用已协商的传输，冷却到期后重新准入 URMA。 + +`example/rdma_performance/recovery_client` 支持 `--transport=urma`，沿用 `--mode`、`--mode_file` 和 CSV 统计，可用于双机故障注入。 diff --git a/docs/en/urma.md b/docs/en/urma.md index a5b80f45b9..29939960b9 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -207,3 +207,32 @@ is tracked as a future enhancement. rejected at `ContextInitOrDie`. - A hardware data path requires a supported UMDK provider and `liburma`. - The current implementation targets Linux. + +## Runtime recovery + +Configure admission for new connections with the process-level API: + +```cpp +#include "brpc/urma/urma_recovery_policy.h" + +brpc::urma::SetUrmaRecoveryCooldownMs(5000); +brpc::urma::SetUrmaRecoveryMode(brpc::urma::UrmaRecoveryMode::AUTO); +``` + +LEGACY is the default. AUTO cools a client peer for five seconds after retry +exhaustion, RNR, JFC errors or WR submission failures. New connections use TCP +during cooldown. TCP_ONLY selects TCP for new connections. Servers decline +URMA through the v2/v3 hello and continue processing TCP requests. + +URMA and RDMA keep separate modes, epochs, cooldown caches and metrics. +Each connection reports its first runtime failure; mode changes filter reports +from older epochs. Access errors are counted separately, flush completions do +not add a root cause, and exhausted SQ/receive credits retain EAGAIN backpressure. +ACK submission failures propagate to connection error handling. + +GetUrmaRecoveryStatus() and /vars/urma_recovery_status expose failure reasons, +cooldown state and negotiation counts. /vars/urma_recovery_tcp_fallback_admissions +counts TCP admissions. Mode changes apply to new connections. Existing connections +keep their negotiated transport; URMA admission resumes after cooldown. + +`example/rdma_performance/recovery_client --transport=urma` supports `--mode`, `--mode_file` and CSV statistics for two-host fault injection. diff --git a/example/rdma_performance/RECOVERY.md b/example/rdma_performance/RECOVERY.md index 9b074ce8c1..ed6c22f73d 100644 --- a/example/rdma_performance/RECOVERY.md +++ b/example/rdma_performance/RECOVERY.md @@ -92,3 +92,11 @@ RNR 测试需要接收端故障注入，暂停补充接收 WR 并突破接收 cr 使用 `--use_rdma=false` 与 TCP 服务端可做无网卡烟测。 客户端退出码 0 表示至少一次 RPC 成功，2 表示整个测试均失败；故障窗口内的失败数单独查看。 验收记录故障前后成功率、恢复时延和各原因计数；性能基线使用原 performance client。 + +## URMA + +链接启用 URMA 的 brpc，服务端配置 `ServerOptions.socket_mode = SOCKET_MODE_URMA`， +客户端增加 `--transport=urma`。`--mode`、`--mode_file`、`--cooldown_ms` 和连接类型沿用上述配置。 +CSV 的 `transport_negotiated` 记录选定传输的协商数，诊断入口使用 `/vars/urma_recovery_*`。 +断开 URMA 数据路径并保持 TCP 可达，观察重试耗尽或 RNR 计数、对端冷却和后续 TCP 调用； +冷却到期后使用 short 连接验证 URMA 重新准入。 diff --git a/example/rdma_performance/recovery_client.cpp b/example/rdma_performance/recovery_client.cpp index 9b4a36badf..171a64f500 100644 --- a/example/rdma_performance/recovery_client.cpp +++ b/example/rdma_performance/recovery_client.cpp @@ -24,6 +24,7 @@ #include "brpc/controller.h" #include "brpc/server.h" #include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "test.pb.h" @@ -32,6 +33,7 @@ DEFINE_string(mode, "auto", "Initial mode: legacy, auto, tcp_only"); DEFINE_string(mode_file, "", "Read a recovery mode from this file while running"); DEFINE_string(connection_type, "short", "short exercises new admission on each RPC"); DEFINE_bool(use_rdma, true, "Use RDMA; false runs a TCP smoke test"); +DEFINE_string(transport, "", "rdma, urma or tcp; empty uses --use_rdma"); DEFINE_int32(seconds, 60, "Duration of the fault-injection run"); DEFINE_int32(interval_ms, 100, "Pause between RPCs"); DEFINE_int32(timeout_ms, 1000, "RPC deadline"); @@ -40,29 +42,43 @@ DEFINE_int32(dummy_port, 8001, "Local /vars port; 0 disables it"); static bool SetMode(const std::string& value) { using brpc::rdma::RdmaRecoveryMode; + RdmaRecoveryMode mode; if (value == "legacy") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::LEGACY); + mode = RdmaRecoveryMode::LEGACY; + } else if (value == "auto") { + mode = RdmaRecoveryMode::AUTO; + } else if (value == "tcp_only") { + mode = RdmaRecoveryMode::TCP_ONLY; + } else { + return false; } - if (value == "auto") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::AUTO); - } - if (value == "tcp_only") { - return brpc::rdma::SetRdmaRecoveryMode(RdmaRecoveryMode::TCP_ONLY); - } - return false; + return FLAGS_transport == "urma" ? brpc::urma::SetUrmaRecoveryMode(mode) : + brpc::rdma::SetRdmaRecoveryMode(mode); } int main(int argc, char* argv[]) { GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + if (FLAGS_transport.empty()) { + FLAGS_transport = FLAGS_use_rdma ? "rdma" : "tcp"; + } + if (FLAGS_transport != "rdma" && FLAGS_transport != "urma" && + FLAGS_transport != "tcp") { + std::cerr << "Invalid transport\n"; + return 1; + } + const bool cooldown_ok = FLAGS_transport == "urma" ? + brpc::urma::SetUrmaRecoveryCooldownMs(FLAGS_cooldown_ms) : + brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms); if (FLAGS_seconds <= 0 || FLAGS_interval_ms < 0 || FLAGS_timeout_ms <= 0 || - !brpc::rdma::SetRdmaRecoveryCooldownMs(FLAGS_cooldown_ms) || + !cooldown_ok || !SetMode(FLAGS_mode)) { std::cerr << "Invalid duration, interval, timeout, cooldown or mode\n"; return 1; } brpc::Channel channel; brpc::ChannelOptions options; - options.socket_mode = FLAGS_use_rdma ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; + options.socket_mode = FLAGS_transport == "urma" ? brpc::SOCKET_MODE_URMA : + FLAGS_transport == "rdma" ? brpc::SOCKET_MODE_RDMA : brpc::SOCKET_MODE_TCP; options.connection_type = FLAGS_connection_type; options.timeout_ms = FLAGS_timeout_ms; options.connect_timeout_ms = FLAGS_timeout_ms; @@ -78,7 +94,7 @@ int main(int argc, char* argv[]) { uint64_t failed = 0; const int64_t start = butil::monotonic_time_us(); int64_t next_report = start; - std::cout << "seconds,mode,success,failure,rdma_negotiated,tcp_fallback_admissions," + std::cout << "seconds,mode,success,failure,transport_negotiated,tcp_fallback_admissions," "cooled_peers,port_active,port_cooldown_ms\n"; while (!brpc::IsAskedToQuit() && butil::monotonic_time_us() - start < static_cast(FLAGS_seconds) * 1000000) { @@ -106,7 +122,8 @@ int main(int argc, char* argv[]) { } const int64_t now = butil::monotonic_time_us(); if (now >= next_report) { - const auto s = brpc::rdma::GetRdmaRecoveryStatus(); + const auto s = FLAGS_transport == "urma" ? + brpc::urma::GetUrmaRecoveryStatus() : brpc::rdma::GetRdmaRecoveryStatus(); std::cout << (now - start) / 1000000 << ',' << mode << ',' << succeeded << ',' << failed << ',' << s.negotiated_connections << ',' << s.tcp_only_admissions + s.port_cooldown_admissions + diff --git a/src/brpc/rdma/rdma_recovery_policy.cpp b/src/brpc/rdma/rdma_recovery_policy.cpp index fbaa4d6fd2..c8329a95a1 100644 --- a/src/brpc/rdma/rdma_recovery_policy.cpp +++ b/src/brpc/rdma/rdma_recovery_policy.cpp @@ -16,6 +16,7 @@ // under the License. #include "brpc/rdma/rdma_recovery_policy.h" +#include "brpc/urma/urma_recovery_policy.h" #include "butil/time.h" #include "bvar/passive_status.h" @@ -34,6 +35,8 @@ const char* RdmaFailureReasonName(RdmaFailureReason reason) { case RdmaFailureReason::CQ_NOTIFY: return "cq_notify"; case RdmaFailureReason::SEND_POST: return "send_post"; case RdmaFailureReason::ACCESS_ERROR: return "access_error"; + case RdmaFailureReason::TRANSPORT_ERROR: return "transport_error"; + case RdmaFailureReason::RECV_POST: return "recv_post"; default: return "unknown"; } } @@ -198,9 +201,10 @@ static void DescribeRecovery(std::ostream& os, void* arg) { } struct RecoveryMetrics { - explicit RecoveryMetrics(RdmaRecoveryPolicy* policy) - : status("rdma_recovery_status", DescribeRecovery, policy) - , fallbacks("rdma_recovery_tcp_fallback_admissions", [](void* arg) { + RecoveryMetrics(RdmaRecoveryPolicy* policy, const char* status_name, + const char* fallback_name) + : status(status_name, DescribeRecovery, policy) + , fallbacks(fallback_name, [](void* arg) { const auto s = static_cast(arg)->GetStatus(); return s.tcp_only_admissions + s.port_cooldown_admissions + s.peer_cooldown_admissions; @@ -214,7 +218,8 @@ RdmaRecoveryPolicy& GlobalRdmaRecoveryPolicy() { // Async event callbacks may outlive ordinary static destructors. static RdmaRecoveryPolicy* policy = new RdmaRecoveryPolicy(butil::monotonic_time_us); - static RecoveryMetrics* metrics = new RecoveryMetrics(policy); + static RecoveryMetrics* metrics = new RecoveryMetrics( + policy, "rdma_recovery_status", "rdma_recovery_tcp_fallback_admissions"); (void)metrics; return *policy; } @@ -236,4 +241,33 @@ RdmaRecoveryStatus GetRdmaRecoveryStatus() { } } // namespace rdma + +namespace urma { + +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy() { + static UrmaRecoveryPolicy* policy = + new UrmaRecoveryPolicy(butil::monotonic_time_us); + static rdma::RecoveryMetrics* metrics = new rdma::RecoveryMetrics( + policy, "urma_recovery_status", "urma_recovery_tcp_fallback_admissions"); + (void)metrics; + return *policy; +} + +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode) { + return GlobalUrmaRecoveryPolicy().SetMode(mode); +} + +UrmaRecoveryMode GetUrmaRecoveryMode() { + return GlobalUrmaRecoveryPolicy().GetMode(); +} + +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms) { + return GlobalUrmaRecoveryPolicy().SetCooldownMs(cooldown_ms); +} + +UrmaRecoveryStatus GetUrmaRecoveryStatus() { + return GlobalUrmaRecoveryPolicy().GetStatus(); +} + +} // namespace urma } // namespace brpc diff --git a/src/brpc/rdma/rdma_recovery_policy.h b/src/brpc/rdma/rdma_recovery_policy.h index 651cf8ccb5..c46c8a0705 100644 --- a/src/brpc/rdma/rdma_recovery_policy.h +++ b/src/brpc/rdma/rdma_recovery_policy.h @@ -31,7 +31,7 @@ enum class RdmaRecoveryMode { LEGACY, AUTO, TCP_ONLY }; enum class RdmaFailureReason { NONE, RETRY_EXHAUSTED, RNR, QP_FATAL, CQ_ERROR, CQ_POLL, - CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, COUNT + CQ_EVENT, CQ_NOTIFY, SEND_POST, ACCESS_ERROR, TRANSPORT_ERROR, RECV_POST, COUNT }; const char* RdmaFailureReasonName(RdmaFailureReason reason); diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 26c7a1b9dc..50e51bbf82 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -168,6 +168,8 @@ UrmaEndpoint::~UrmaEndpoint() { void UrmaEndpoint::Reset() { DeallocateResources(); _state = UNINIT; + _recovery_epoch = 0; + _runtime_failure.store(UrmaFailureReason::NONE, butil::memory_order_relaxed); _handshake_version = 0; _remote_recv_block_size = 0; _local_window_capacity = 0; @@ -770,6 +772,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { << ", configured_max_sge=" << GetUrmaMaxSge() << ", payload_size=" << this_len << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = rc; return -1; } @@ -815,6 +818,7 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { _resource->jetty->remote_jetty) << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::RECV_POST); errno = status; return -1; } @@ -908,6 +912,7 @@ int UrmaEndpoint::SendImm(uint32_t imm) { << _resource->remote_jetty->id.id << " state=" << GetStateStr() << " on " << _socket->description(); + ReportFailure(UrmaFailureReason::SEND_POST); errno = status; return -1; } @@ -928,6 +933,26 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { LOG(WARNING) << "URMA completion failed, status=" << cr.status; + switch (cr.status) { + case URMA_CR_ACK_TIMEOUT_ERR: + ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); + break; + case URMA_CR_RNR_RETRY_CNT_EXC_ERR: + ReportFailure(UrmaFailureReason::RNR); + break; + case URMA_CR_LOC_ACCESS_ERR: + case URMA_CR_REM_ACCESS_ABORT_ERR: + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + break; + case URMA_CR_WR_FLUSH_ERR: + case URMA_CR_WR_FLUSH_ERR_DONE: + case URMA_CR_WR_SUSPEND_DONE: + case URMA_CR_WR_UNHANDLED: + break; + default: + ReportFailure(UrmaFailureReason::TRANSPORT_ERROR); + break; + } errno = EIO; return -1; } @@ -946,8 +971,7 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { return -1; } _sq_imm_window_size += 1; - SendAck(0); - return 0; + return SendAck(0); } uint16_t wnd = 1; // We signal every WR (complete_enable=1). uint16_t old = @@ -1035,8 +1059,8 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { if (PostRecv(1, zerocopy) < 0) { return -1; } - if (cr.completion_len > 0) { - SendAck(1); + if (cr.completion_len > 0 && SendAck(1) < 0) { + return -1; } return static_cast(cr.completion_len); } @@ -1118,6 +1142,7 @@ void UrmaEndpoint::PollCq(Socket* m) { const int cnt = urma_poll_jfc(ep->_resource->jfc, n, crs); if (cnt < 0) { + ep->ReportFailure(UrmaFailureReason::CQ_POLL); return EIO; } if (cnt == 0) { @@ -1307,6 +1332,21 @@ void UrmaEndpoint::FallbackToTcp(UrmaTransport* transport, bool process_tcp) { } } +void UrmaEndpoint::ReportFailure(UrmaFailureReason reason) { + if (_state.load(butil::memory_order_acquire) != ESTABLISHED) { + return; + } + UrmaFailureReason expected = UrmaFailureReason::NONE; + if (reason == UrmaFailureReason::NONE || + !_runtime_failure.compare_exchange_strong(expected, reason)) { + return; + } + GlobalUrmaRecoveryPolicy().ReportPeerFailure( + _socket->CreatedByConnect() ? + butil::endpoint2str(_socket->remote_side()).c_str() : "", + _recovery_epoch, reason); +} + void UrmaEndpoint::FailHandshake(UrmaTransport* transport, int error, const char* reason) { LOG(ERROR) << "URMA handshake failed in state=" << GetStateStr() @@ -1333,7 +1373,9 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { SocketUniquePtr s(ep->_socket); auto* tp = static_cast(s->_transport.get()); UrmaConnect::RunGuard guard(static_cast(s->_app_connect.get())); - if (!IsUrmaAvailable()) { + if (!IsUrmaAvailable() || + !GlobalUrmaRecoveryPolicy().AllowNewConnection( + butil::endpoint2str(s->remote_side()).c_str(), &ep->_recovery_epoch)) { ep->FallbackToTcp(tp, true); return nullptr; } @@ -1385,6 +1427,7 @@ void* UrmaEndpoint::ProcessHandshakeAtClient(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); return nullptr; } @@ -1420,6 +1463,16 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { ep->FailHandshake(tp, EPROTO, "invalid client hello"); return nullptr; } + if (!IsUrmaAvailable() || + !GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch)) { + tp->_urma_state = UrmaTransport::URMA_OFF; + if (hs->SendLocalHello() < 0) { + ep->FailHandshake(tp, errno ? errno : EIO, "send TCP fallback hello"); + return nullptr; + } + ep->FallbackToTcp(tp, true); + return nullptr; + } ep->_state = S_ALLOC_RES; if (ep->AllocateResources() < 0) { const int saved_errno = errno ? errno : EIO; @@ -1470,6 +1523,7 @@ void* UrmaEndpoint::ProcessHandshakeAtServer(void* arg) { } tp->_urma_state = UrmaTransport::URMA_ON; ep->_state = ESTABLISHED; + GlobalUrmaRecoveryPolicy().RecordNegotiatedConnection(ep->_recovery_epoch); ep->DispatchReceivedBytes(s, 0); } else { ep->FallbackToTcp(tp, true); @@ -1560,6 +1614,8 @@ void UrmaEndpoint::DebugInfo(std::ostream& os, butil::StringPiece) const { << " remote_recv_block_size=" << _remote_recv_block_size << " sq_window=" << _sq_window_size.load(butil::memory_order_relaxed) << " remote_rq_window=" << _remote_rq_window_size.load(butil::memory_order_relaxed) + << " recovery_epoch=" << _recovery_epoch + << " runtime_failure=" << rdma::RdmaFailureReasonName(_runtime_failure.load()) << " handshake_version=" << _handshake_version << " read_buf=" << _input_processor.read_buf().size(); } @@ -1579,6 +1635,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, const int saved_errno = errno; PLOG(ERROR) << "Fail to wait URMA JFC event from " << s->description(); + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(saved_errno, "Fail to wait URMA JFC event: %s", berror(saved_errno)); return -1; @@ -1589,6 +1646,7 @@ int UrmaEndpoint::WaitCqEvent(SocketUniquePtr& s, if (*event_jfc != _resource->jfc) { LOG(ERROR) << "Unexpected URMA JFC event on " << s->description(); errno = EPROTO; + ReportFailure(UrmaFailureReason::CQ_EVENT); s->SetFailed(EPROTO, "Unexpected URMA JFC event"); return -1; } @@ -1604,6 +1662,7 @@ int UrmaEndpoint::ReqNotifyCq() { if (rc != URMA_SUCCESS) { errno = rc; PLOG(WARNING) << "Fail to rearm URMA JFC"; + ReportFailure(UrmaFailureReason::CQ_NOTIFY); _socket->SetFailed(rc, "Fail to rearm URMA JFC: %s", berror(rc)); return -1; } diff --git a/src/brpc/urma/urma_endpoint.h b/src/brpc/urma/urma_endpoint.h index e5aed5ac66..5d2a3908b0 100644 --- a/src/brpc/urma/urma_endpoint.h +++ b/src/brpc/urma/urma_endpoint.h @@ -18,6 +18,8 @@ #ifndef BRPC_URMA_ENDPOINT_H #define BRPC_URMA_ENDPOINT_H +#include "brpc/urma/urma_recovery_policy.h" + #include #include #include @@ -100,6 +102,7 @@ struct UrmaResource { // windows, and the registered buffer bookkeeping. Cache-line padded to avoid // false sharing between connections. class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { + friend class UrmaRecoveryTestPeer; friend class UrmaConnect; friend class Socket; friend class UrmaTransport; @@ -261,6 +264,7 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { inline void TryReadOnTcp(); void FallbackToTcp(UrmaTransport* transport, bool process_tcp); + void ReportFailure(UrmaFailureReason reason); void FailHandshake(UrmaTransport* transport, int error, const char* reason); @@ -272,6 +276,9 @@ class BAIDU_CACHELINE_ALIGNMENT UrmaEndpoint : public SocketUser { // Not owned. Socket* _socket; + uint64_t _recovery_epoch{0}; + butil::atomic _runtime_failure{UrmaFailureReason::NONE}; + // Handshake state. butil::atomic _state{UNINIT}; int _handshake_version{0}; // 0 = unnegotiated; 2 = v2; 3 = v3 diff --git a/src/brpc/urma/urma_handshake.cpp b/src/brpc/urma/urma_handshake.cpp index b7018606e0..ac7eb115ce 100644 --- a/src/brpc/urma/urma_handshake.cpp +++ b/src/brpc/urma/urma_handshake.cpp @@ -328,7 +328,12 @@ int UrmaHandshakeServerV3::ReceiveAndParseRemoteHello(ParsedHello* out, int UrmaHandshakeServerV3::SendLocalHello() { UrmaHello msg; _ep->FillLocalHelloV3(&msg); - // v3 has no zero-out path; the client rejects jetty_id==0 via ValidHello. + auto* tp = static_cast(_ep->_socket->_transport.get()); + if (tp->_urma_state.load(butil::memory_order_acquire) == + UrmaTransport::URMA_OFF) { + msg.set_jetty_id(0); + msg.set_buffer_size(0); + } return _ep->WriteHelloV3(msg); } diff --git a/src/brpc/urma/urma_recovery_policy.h b/src/brpc/urma/urma_recovery_policy.h new file mode 100644 index 0000000000..da2792fd54 --- /dev/null +++ b/src/brpc/urma/urma_recovery_policy.h @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#ifndef BRPC_URMA_RECOVERY_POLICY_H +#define BRPC_URMA_RECOVERY_POLICY_H + +#include "brpc/rdma/rdma_recovery_policy.h" + +namespace brpc { +namespace urma { + +// Share the admission algorithm while keeping transport state independent. +using UrmaRecoveryMode = rdma::RdmaRecoveryMode; +using UrmaFailureReason = rdma::RdmaFailureReason; +using UrmaRecoveryStatus = rdma::RdmaRecoveryStatus; +using UrmaRecoveryPolicy = rdma::RdmaRecoveryPolicy; + +UrmaRecoveryPolicy& GlobalUrmaRecoveryPolicy(); +bool SetUrmaRecoveryMode(UrmaRecoveryMode mode); +UrmaRecoveryMode GetUrmaRecoveryMode(); +bool SetUrmaRecoveryCooldownMs(int64_t cooldown_ms); +UrmaRecoveryStatus GetUrmaRecoveryStatus(); + +} // namespace urma +} // namespace brpc + +#endif // BRPC_URMA_RECOVERY_POLICY_H diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index d27367a504..21b76dbb6a 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -16,6 +16,8 @@ // under the License. #include +#include +#include #include #include #include @@ -28,6 +30,15 @@ #include "brpc/urma/urma_handshake.pb.h" #include "brpc/urma/urma_helper.h" #include "urma_types.h" +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/input_messenger.h" +#include "brpc/urma/urma_endpoint.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "butil/fd_guard.h" +#include "bvar/variable.h" +#include "echo.pb.h" using namespace brpc; @@ -593,6 +604,331 @@ TEST_F(UrmaMockTest, urma_free_device_list(devices); } + +namespace brpc { +namespace urma { + +class UrmaRecoveryTestPeer { +public: + static void Establish(UrmaEndpoint* ep) { + GlobalUrmaRecoveryPolicy().AllowNewConnection("", &ep->_recovery_epoch); + ep->_state = UrmaEndpoint::ESTABLISHED; + } + static ssize_t Complete(UrmaEndpoint* ep, urma_cr_status_t status) { + urma_cr_t cr{}; + cr.status = status; + return ep->HandleCompletion(cr); + } + static void Report(UrmaEndpoint* ep, UrmaFailureReason reason) { + ep->ReportFailure(reason); + } + static void PollFailure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jfc_t invalid_jfc{}; + resource.jfc = &invalid_jfc; + ep->_resource = &resource; + SocketOptions options; + options.user = ep; + SocketId sid; + if (Socket::Create(options, &sid) == 0) { + SocketUniquePtr carrier; + if (Socket::Address(sid, &carrier) == 0) { + const bool polling = FLAGS_urma_use_polling; + FLAGS_urma_use_polling = true; + UrmaEndpoint::PollCq(carrier.get()); + FLAGS_urma_use_polling = polling; + carrier->SetFailed(); + } + } + ep->_resource = nullptr; + resource.jfc = nullptr; + } + static bool Backpressure(UrmaEndpoint* ep) { + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_window_size = 0; + ep->_remote_rq_window_size = 1; + butil::IOBuf data; + data.append("payload"); + butil::IOBuf* buf = &data; + const ssize_t rc = ep->CutFromIOBufList(&buf, 1); + const bool ok = rc == -1 && errno == EAGAIN && data.size() == 7; + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return ok; + } + static int FailedAck(UrmaEndpoint* ep) { + // A Jetty without an imported target fails provider submission. + UrmaResource resource; + urma_jetty_t jetty{}; + urma_target_jetty_t remote{}; + resource.jetty = &jetty; + resource.remote_jetty = &remote; + ep->_resource = &resource; + ep->_sq_imm_window_size = 1; + ep->_remote_window_capacity = 2; + ep->_new_rq_wrs = 2; + urma_cr_t cr{}; + cr.status = URMA_CR_SUCCESS; + cr.user_ctx = 0; + const int rc = ep->HandleCompletion(cr); + ep->_resource = nullptr; + resource.jetty = nullptr; + resource.remote_jetty = nullptr; + return rc; + } +}; + +} // namespace urma +} // namespace brpc + +class UrmaRecoveryTest : public UrmaMockTest { +protected: + void SetUp() override { + UrmaMockTest::SetUp(); + if (IsSkipped() || HasFatalFailure()) { + return; + } + urma::g_skip_urma_init = true; + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + urma::GlobalUrmaRecoveryPolicy().ReportPortState(true); + } + void TearDown() override { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + rdma::SetRdmaRecoveryMode(rdma::RdmaRecoveryMode::LEGACY); + UrmaMockTest::TearDown(); + } + void CreateSocket(SocketUniquePtr* socket) { + SocketOptions opt; + butil::str2endpoint("127.0.0.1:8123", &opt.remote_side); + opt.user = get_client_side_messenger(); + SocketId sid; + ASSERT_EQ(0, Socket::Create(opt, &sid)); + ASSERT_EQ(0, Socket::Address(sid, socket)); + } +}; + +TEST_F(UrmaRecoveryTest, independent_transport_modes_and_metrics) { + ASSERT_TRUE(urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_TRUE(rdma::GlobalRdmaRecoveryPolicy().AllowNewConnection("peer")); + EXPECT_EQ(rdma::RdmaRecoveryMode::LEGACY, rdma::GetRdmaRecoveryMode()); + EXPECT_FALSE(urma::SetUrmaRecoveryCooldownMs(0)); + EXPECT_TRUE(urma::SetUrmaRecoveryCooldownMs(5000)); + EXPECT_NE(std::string::npos, + bvar::Variable::describe_exposed("urma_recovery_status").find("tcp_only")); + EXPECT_FALSE(bvar::Variable::describe_exposed( + "urma_recovery_tcp_fallback_admissions").empty()); +} + +TEST_F(UrmaRecoveryTest, completion_errors_cool_peer_once) { + for (auto status : {URMA_CR_ACK_TIMEOUT_ERR, URMA_CR_RNR_RETRY_CNT_EXC_ERR, + URMA_CR_LOC_ACCESS_ERR, URMA_CR_REM_OPERATION_ERR}) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::LEGACY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, status)); + const auto reason = status == URMA_CR_ACK_TIMEOUT_ERR ? + urma::UrmaFailureReason::RETRY_EXHAUSTED : + status == URMA_CR_RNR_RETRY_CNT_EXC_ERR ? urma::UrmaFailureReason::RNR : + status == URMA_CR_LOC_ACCESS_ERR ? urma::UrmaFailureReason::ACCESS_ERROR : + urma::UrmaFailureReason::TRANSPORT_ERROR; + EXPECT_EQ(before.failures[static_cast(reason)] + 1, + urma::GetUrmaRecoveryStatus().failures[static_cast(reason)]); + EXPECT_EQ(status == URMA_CR_LOC_ACCESS_ERR, + urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)], + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); + socket->SetFailed(); + } +} + +TEST_F(UrmaRecoveryTest, flush_backpressure_and_old_epoch_do_not_cool_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_TRUE(urma::UrmaRecoveryTestPeer::Backpressure(&ep)); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete(&ep, URMA_CR_WR_FLUSH_ERR)); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + urma::UrmaRecoveryTestPeer::Report(&ep, urma::UrmaFailureReason::CQ_POLL); + EXPECT_TRUE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, ack_post_failure_propagates) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::FailedAck(&ep)); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + socket->SetFailed(); +} + +TEST_F(UrmaRecoveryTest, jfc_poll_failure_fails_socket_and_cools_peer) { + SocketUniquePtr socket; + ASSERT_NO_FATAL_FAILURE(CreateSocket(&socket)); + urma::UrmaEndpoint ep(socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&ep); + const auto before = urma::GetUrmaRecoveryStatus(); + urma::UrmaRecoveryTestPeer::PollFailure(&ep); + EXPECT_TRUE(socket->Failed()); + EXPECT_FALSE(urma::GlobalUrmaRecoveryPolicy().AllowNewConnection("127.0.0.1:8123")); + EXPECT_EQ(before.failures[static_cast(urma::UrmaFailureReason::CQ_POLL)] + 1, + urma::GetUrmaRecoveryStatus().failures[ + static_cast(urma::UrmaFailureReason::CQ_POLL)]); +} + +class UrmaRecoveryEcho : public ::test::EchoService { + void Echo(google::protobuf::RpcController*, const ::test::EchoRequest* request, + ::test::EchoResponse* response, google::protobuf::Closure* done) override { + response->set_message(request->message()); + done->Run(); + } +}; + +TEST_F(UrmaRecoveryTest, tcp_only_client_completes_rpc_on_urma_server) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + UrmaRecoveryEcho service; + Server server; + ASSERT_EQ(0, server.AddService(&service, SERVER_DOESNT_OWN_SERVICE)); + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + Channel channel; + ChannelOptions co; + co.socket_mode = SOCKET_MODE_URMA; + co.connection_type = "short"; + co.timeout_ms = 2000; + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + ::test::EchoService_Stub stub(&channel); + Controller cntl; + ::test::EchoRequest request; + ::test::EchoResponse response; + request.set_message("recovery over TCP"); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::AUTO); + SocketOptions failed_options; + failed_options.remote_side = server.listen_address(); + failed_options.user = get_client_side_messenger(); + SocketId failed_sid; + ASSERT_EQ(0, Socket::Create(failed_options, &failed_sid)); + SocketUniquePtr failed_socket; + ASSERT_EQ(0, Socket::Address(failed_sid, &failed_socket)); + { + urma::UrmaEndpoint failed_ep(failed_socket.get()); + urma::UrmaRecoveryTestPeer::Establish(&failed_ep); + EXPECT_EQ(-1, urma::UrmaRecoveryTestPeer::Complete( + &failed_ep, URMA_CR_RNR_RETRY_CNT_EXC_ERR)); + } + failed_socket->SetFailed(); + const auto before = urma::GetUrmaRecoveryStatus(); + cntl.Reset(); + response.Clear(); + stub.Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(request.message(), response.message()); + EXPECT_EQ(before.peer_cooldown_admissions + 1, + urma::GetUrmaRecoveryStatus().peer_cooldown_admissions); + server.Stop(0); + server.Join(); +} + +TEST_F(UrmaRecoveryTest, server_declines_v2_and_v3_without_allocating_jetty) { + urma::SetUrmaRecoveryMode(urma::UrmaRecoveryMode::TCP_ONLY); + Server server; + ServerOptions so; + so.socket_mode = SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + for (int version : {2, 3}) { + butil::fd_guard fd(butil::tcp_connect(server.listen_address(), nullptr)); + ASSERT_GE(fd, 0); + timeval timeout{2, 0}; + ASSERT_EQ(0, setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout))); + urma::v2_wire::HelloMessage hello; + hello.msg_len = urma::v2_wire::HELLO_PACKET_LEN; + hello.hello_ver = urma::v2_wire::HELLO_V2_VERSION; + hello.impl_ver = urma::v2_wire::IMPL_V2_VERSION; + hello.buffer_size = 8192; + hello.recv_buffer_cnt = 127; + hello.jetty_id = 42; + memset(hello.eid, 1, 16); + memset(hello.seg_eid, 1, 16); + hello.tp_type = URMA_CTP; + hello.seg_va = 0x100000; + hello.seg_len = 1024 * 1024; + if (version == 2) { + uint8_t packet[urma::v2_wire::HELLO_PACKET_LEN]; + memcpy(packet, "URMA", 4); + hello.Serialize(packet + 4); + ASSERT_EQ(sizeof(packet), static_cast(write(fd, packet, sizeof(packet)))); + uint8_t reply[sizeof(packet)]; + ASSERT_EQ(sizeof(reply), static_cast(recv(fd, reply, sizeof(reply), MSG_WAITALL))); + EXPECT_EQ(0, memcmp(reply, "URMA", 4)); + urma::v2_wire::HelloMessage rejected; + rejected.Deserialize(reply + 4); + EXPECT_EQ(0, rejected.hello_ver); + EXPECT_EQ(0u, rejected.jetty_id); + } else { + urma::UrmaHello msg; + msg.set_buffer_size(8192); + msg.set_recv_buffer_cnt(127); + msg.set_jetty_id(42); + msg.set_uasid(0); + msg.set_seg_uasid(0); + msg.set_seg_token_id(0); + msg.set_eid(hello.eid, 16); + msg.set_seg_eid(hello.seg_eid, 16); + msg.set_tp_type(URMA_CTP); + msg.set_seg_va(hello.seg_va); + msg.set_seg_len(hello.seg_len); + const std::string body = msg.SerializeAsString(); + uint32_t size = butil::HostToNet32(body.size()); + std::string packet("URM3", 4); + packet.append(reinterpret_cast(&size), 4); + packet.append(body); + ASSERT_EQ(packet.size(), static_cast(write(fd, packet.data(), packet.size()))); + char header[8]; + ASSERT_EQ(8, recv(fd, header, 8, MSG_WAITALL)); + EXPECT_EQ(0, memcmp(header, "URM3", 4)); + memcpy(&size, header + 4, 4); + size = butil::NetToHost32(size); + ASSERT_LE(size, 4096u); + std::string reply(size, '\0'); + ASSERT_EQ(size, static_cast(recv(fd, &reply[0], size, MSG_WAITALL))); + urma::UrmaHello rejected; + ASSERT_TRUE(rejected.ParseFromString(reply)); + EXPECT_EQ(0u, rejected.jetty_id()); + } + const std::string request = "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + ASSERT_EQ(request.size(), static_cast(write(fd, request.data(), request.size()))); + char response[4096]; + const ssize_t nr = recv(fd, response, sizeof(response), 0); + ASSERT_GT(nr, 0); + EXPECT_NE(std::string::npos, std::string(response, nr).find(" 200 ")); + } + server.Stop(0); + server.Join(); +} + #else // BRPC_WITH_URMA // When URMA is not compiled in, the test file is a no-op so the build stays From c371849eb7bd01c4ae149f1fb74c14648d44faba Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:01:51 -0700 Subject: [PATCH 6/8] Add URMA memory access failure diagnostics --- src/brpc/urma/mock_urma.cpp | 8 +++++++- src/brpc/urma/urma_endpoint.cpp | 20 ++++++++++++++++++-- src/brpc/urma/urma_helper.cpp | 6 ++++++ 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/brpc/urma/mock_urma.cpp b/src/brpc/urma/mock_urma.cpp index b0572c2e86..a1844d05e4 100644 --- a/src/brpc/urma/mock_urma.cpp +++ b/src/brpc/urma/mock_urma.cpp @@ -390,13 +390,19 @@ urma_target_seg_t *urma_register_seg(urma_context_t *ctx, urma_seg_cfg_t *cfg) { if (!ctx || !cfg || context_map.find(ctx) == context_map.end()) { return nullptr; } - urma_target_seg_t *seg = new urma_target_seg_t; + urma_target_seg_t *seg = new urma_target_seg_t{}; memset(&seg->seg.ubva.eid, 0, sizeof(urma_eid_t)); seg->seg.ubva.eid.raw[0] = 1; seg->seg.ubva.uasid = 0; seg->seg.ubva.va = cfg->va; seg->seg.len = cfg->len; seg->seg.token_id = cfg->token_value.token; + seg->seg.attr.bs.access = cfg->flag.bs.access; + seg->seg.attr.bs.token_policy = cfg->flag.bs.token_policy; + seg->seg.attr.bs.cacheable = cfg->flag.bs.cacheable; + seg->user_ctx = cfg->user_ctx; + seg->urma_ctx = ctx; + seg->token_id = cfg->token_id; seg_map[seg] = 1; return seg; } diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 50e51bbf82..0b4771ac9b 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -800,7 +800,8 @@ int UrmaEndpoint::DoPostRecv(void* block, size_t block_size) { urma_sge_t sge{reinterpret_cast(block), static_cast(block_size), tseg, nullptr}; urma_sg_t sg{&sge, 1}; - urma_jfr_wr_t wr{sg, 0, nullptr}; + // Preserve the receive address in completions for access-error diagnostics. + urma_jfr_wr_t wr{sg, reinterpret_cast(block), nullptr}; urma_jfr_wr_t* bad = nullptr; // Use the shared-JFR path on every device, including bonding. The bonding // provider owns physical receive scheduling for the JFR; a local @@ -932,7 +933,22 @@ int UrmaEndpoint::SendAck(int num) { ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { bool zerocopy = FLAGS_urma_recv_zerocopy; if (cr.status != URMA_CR_SUCCESS) { - LOG(WARNING) << "URMA completion failed, status=" << cr.status; + LOG(WARNING) << "URMA completion failed, status=" << cr.status + << " direction=" << (cr.flag.bs.s_r ? "recv" : "send") + << " flags=" << static_cast(cr.flag.value) + << " opcode=" << cr.opcode + << " local_id=" << cr.local_id + << " tpn=" << cr.tpn + << " user_ctx=" << cr.user_ctx + << " completion_len=" << cr.completion_len + << " jetty_id=" + << (_resource && _resource->jetty ? + _resource->jetty->jetty_id.id : 0) + << " jfr_id=" + << (_resource && _resource->jfr ? + _resource->jfr->jfr_id.id : 0) + << " state=" << GetStateStr() + << " on " << _socket->description(); switch (cr.status) { case URMA_CR_ACK_TIMEOUT_ERR: ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index ded662fb22..d9dd06bc72 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -383,6 +383,12 @@ bool InitPool() { g_pool_base = nullptr; return false; } + LOG(INFO) << "URMA registered buffer pool: base=" << g_pool_base + << " size=" << g_pool_size + << " segment_va=" << g_pool_seg->seg.ubva.va + << " segment_len=" << g_pool_seg->seg.len + << " segment_access=" << g_pool_seg->seg.attr.bs.access + << " segment_token_id=" << g_pool_seg->seg.token_id; g_pool->in_use.assign(count, 0); for (size_t i = 0; i < count; ++i) { auto shard = i % kShardCount; From d87dfe891173fef7394f556395d10575fea325b2 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:58:49 -0700 Subject: [PATCH 7/8] Initialize and validate URMA send descriptors --- src/brpc/urma/urma_endpoint.cpp | 40 +++++++++++++++++++++++++++++---- src/brpc/urma/urma_helper.cpp | 21 +++++++++++++++++ src/brpc/urma/urma_helper.h | 4 ++++ test/brpc_urma_unittest.cpp | 38 +++++++++++++++++++++++++++++++ 4 files changed, 99 insertions(+), 4 deletions(-) diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 0b4771ac9b..2545a69987 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -663,9 +663,16 @@ class UrmaIOBuf : private butil::IOBuf { if (len + this_len > max_len) { this_len = max_len - len; } - sglist[*sge_index].addr = reinterpret_cast(start); - sglist[*sge_index].len = static_cast(this_len); - sglist[*sge_index].tseg = tseg; + if (!InitUrmaLocalSge(&sglist[*sge_index], + reinterpret_cast(start), this_len, tseg)) { + LOG(ERROR) << "Invalid URMA send SGE: address=" + << reinterpret_cast(start) + << " length=" << this_len + << " segment_va=" << tseg->seg.ubva.va + << " segment_len=" << tseg->seg.len + << " context_matches=" << (tseg->urma_ctx == GetUrmaContext()); + return -1; + } cutn(to, this_len); len += this_len; (*sge_index)++; @@ -719,6 +726,9 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ssize_t n = data->cut_into_sglist(sglist, &sge_index, to, max_sge, max_len - this_len); if (n < 0) { + if (errno == ERDMAMEM) { + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + } return -1; } this_len += n; @@ -738,7 +748,8 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { wr.flag.bs.complete_enable = 1; wr.tjetty = _resource->remote_jetty; wr.send.src = sg; - wr.user_ctx = 1; + // Zero identifies credit ACKs; data completions identify their SQ slot. + wr.user_ctx = static_cast(_sq_current) + 1; urma_jfs_wr_t* bad_wr = nullptr; const uint16_t sq_slot = _sq_current; const uint32_t local_jetty_id = _resource->jetty->jetty_id.id; @@ -949,6 +960,27 @@ ssize_t UrmaEndpoint::HandleCompletion(const urma_cr_t& cr) { _resource->jfr->jfr_id.id : 0) << " state=" << GetStateStr() << " on " << _socket->description(); + if (!cr.flag.bs.s_r && cr.user_ctx > 0 && cr.user_ctx <= _sbuf.size()) { + const size_t slot = static_cast(cr.user_ctx - 1); + const auto& buffer = _sbuf[slot]; + LOG(WARNING) << "URMA failed SEND buffer: slot=" << slot + << " bytes=" << buffer.size() + << " blocks=" << buffer.backing_block_num() + << " on " << _socket->description(); + for (size_t i = 0; i < buffer.backing_block_num(); ++i) { + const auto block = buffer.backing_block(i); + auto* segment = GetPoolSegFor(const_cast(block.data())); + LOG(WARNING) << "URMA failed SEND block: slot=" << slot + << " block=" << i + << " address=" << reinterpret_cast(block.data()) + << " length=" << block.size() + << " pool_segment=" << (segment != nullptr) + << " segment_va=" << (segment ? segment->seg.ubva.va : 0) + << " segment_len=" << (segment ? segment->seg.len : 0) + << " segment_token_id=" << (segment ? segment->seg.token_id : 0) + << " on " << _socket->description(); + } + } switch (cr.status) { case URMA_CR_ACK_TIMEOUT_ERR: ReportFailure(UrmaFailureReason::RETRY_EXHAUSTED); diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index d9dd06bc72..f7c94d4827 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -28,6 +28,7 @@ #include #include +#include #include #include #include @@ -418,6 +419,26 @@ urma_target_seg_t* GetPoolSegFor(void* buf) { return nullptr; } +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment) { + if (!sge || !segment || length == 0 || + length > std::numeric_limits::max() || + segment->urma_ctx != GetUrmaContext()) { + errno = ERDMAMEM; + return false; + } + const uint64_t base = segment->seg.ubva.va; + const uint64_t size = segment->seg.len; + if (size > std::numeric_limits::max() - base || + address < base || address - base > size || + length > size - (address - base)) { + errno = ERDMAMEM; + return false; + } + *sge = urma_sge_t{address, static_cast(length), segment, nullptr}; + return true; +} + static void GlobalRelease() { g_urma_available.store(false, butil::memory_order_release); if (g_mem_alloc_orig) { diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index 871f1a6354..c7439812d7 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -83,6 +83,10 @@ int FindUrmaPriorityForTpType(const urma_device_attr_t& attr, // Return the priority selected for the CTP jettys created by brpc. uint8_t GetUrmaJettyPriority(); +// Initialize a local SGE whose complete byte range belongs to this context. +bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, + urma_target_seg_t* segment); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index 21b76dbb6a..da77a94d48 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -274,6 +274,44 @@ TEST(UrmaHelperTest, selects_priority_matching_transport_path_type) { EXPECT_EQ(-1, urma::FindUrmaPriorityForTpType(attr, URMA_UTP)); } +TEST(UrmaHelperTest, local_sge_initializes_optional_fields_and_accepts_segment_edges) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge; + memset(&sge, 0xff, sizeof(sge)); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x1000, 8192, &segment)); + EXPECT_EQ(0x1000u, sge.addr); + EXPECT_EQ(8192u, sge.len); + EXPECT_EQ(&segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + ASSERT_TRUE(urma::InitUrmaLocalSge(&sge, 0x2fff, 1, &segment)); + EXPECT_EQ(0x2fffu, sge.addr); + EXPECT_EQ(1u, sge.len); +} + +TEST(UrmaHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { + urma_target_seg_t segment{}; + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = 0x1000; + segment.seg.len = 8192; + urma_sge_t sge{}; + for (auto values : {std::pair(0xfff, 1), + {0x3000, 1}, {0x2fff, 2}, {0x1000, 0}, + {0x1000, static_cast(UINT32_MAX) + 1}}) { + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, values.first, values.second, &segment)); + EXPECT_EQ(ERDMAMEM, errno); + } + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, nullptr)); + segment.urma_ctx = reinterpret_cast(1); + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, 0x1000, 1, &segment)); + segment.urma_ctx = urma::GetUrmaContext(); + segment.seg.ubva.va = UINT64_MAX - 7; + segment.seg.len = 16; + EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); +} + // --------------------------------------------------------------------------- // SupportedByUrma: only baidu_std. // --------------------------------------------------------------------------- From 82806c03ee976d094026432a044ae4ec6aadbfb1 Mon Sep 17 00:00:00 2001 From: Spicy-cream <299359524+Spicy-cream@users.noreply.github.com> Date: Wed, 28 Oct 2026 00:25:03 -0700 Subject: [PATCH 8/8] Add aligned copied URMA sends for access fault diagnosis --- docs/cn/urma.md | 8 ++ docs/en/urma.md | 10 +++ src/brpc/urma/urma_endpoint.cpp | 23 +++++- src/brpc/urma/urma_helper.cpp | 42 ++++++++++ src/brpc/urma/urma_helper.h | 9 +++ test/brpc_urma_copy_unittest.cpp | 134 +++++++++++++++++++++++++++++++ test/brpc_urma_unittest.cpp | 67 ++++++++++++++++ 7 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 test/brpc_urma_copy_unittest.cpp diff --git a/docs/cn/urma.md b/docs/cn/urma.md index c207e7d7a1..c744bffb32 100644 --- a/docs/cn/urma.md +++ b/docs/cn/urma.md @@ -157,6 +157,7 @@ block 都由注册 segment 支撑，发送路径可直接从 IOBuf block refs | `--urma_rq_size` | 128 | 本地 JFR 深度 [16, 4096] | | `--urma_cqe_poll_once` | 32 | 每次 `urma_poll_jfc` 的上限 | | `--urma_recv_zerocopy` | true | 大于 `--urma_zerocopy_min_size` 的接收零拷贝 | +| `--urma_copy_send` | false | 将主机发送数据复制合并到 64 字节对齐的注册缓冲区 | | `--urma_zerocopy_min_size` | 512 | 小于此值的接收拷贝 | | `--urma_device` | "" | URMA 设备名（空=首个） | | `--urma_max_sge` | 0 | 每 WR SGE 上限（0=设备上限） | @@ -218,3 +219,10 @@ SQ/接收 credit 耗尽保留 EAGAIN 背压，ACK 提交失败向连接错误处 模式切换作用于新连接；存量连接使用已协商的传输，冷却到期后重新准入 URMA。 `example/rdma_performance/recovery_client` 支持 `--transport=urma`，沿用 `--mode`、`--mode_file` 和 CSV 统计，可用于双机故障注入。 + +## 发送访问错误对照 + +设置 `--urma_copy_send=true` 后，每个数据 WR 使用一个新的、64 字节对齐的注册缓冲区， +该缓冲区由发送槽位持有至完成；消息和附件内容保持一致。此模式增加一次 CPU 拷贝。 +可在两端保持设备、队列、消息大小等参数一致，比较原发送路径与复制路径的结果。 +启动日志显示 `copy_send`，失败日志显示完成方向及发送槽位的地址、长度和注册段范围。 diff --git a/docs/en/urma.md b/docs/en/urma.md index 29939960b9..c02e3d6c0b 100644 --- a/docs/en/urma.md +++ b/docs/en/urma.md @@ -168,6 +168,7 @@ All flags use the `urma_` prefix (mirroring RDMA's `rdma_` prefix): | `--urma_rq_size` | 128 | Local JFR depth [16, 4096] | | `--urma_cqe_poll_once` | 32 | Max CQEs per `urma_poll_jfc` | | `--urma_recv_zerocopy` | true | Zero-copy receives above `--urma_zerocopy_min_size` | +| `--urma_copy_send` | false | Copy and coalesce host payloads into a 64-byte aligned registered send buffer | | `--urma_zerocopy_min_size` | 512 | Receives smaller than this are copied | | `--urma_device` | "" | URMA device name (empty = first) | | `--urma_max_sge` | 0 | Max SGEs per WR (0 = device max) | @@ -236,3 +237,12 @@ counts TCP admissions. Mode changes apply to new connections. Existing connectio keep their negotiated transport; URMA admission resumes after cooldown. `example/rdma_performance/recovery_client --transport=urma` supports `--mode`, `--mode_file` and CSV statistics for two-host fault injection. + +## Comparing send access failures + +With `--urma_copy_send=true`, each data WR uses a fresh, 64-byte aligned registered +buffer held by its send slot until completion. Message and attachment contents are +preserved. This mode adds a CPU copy. Compare it with the original send path while +keeping device, queue and payload settings the same on both peers. Startup logs +include `copy_send`; failure logs include direction, send-slot addresses, lengths +and registered segment ranges. diff --git a/src/brpc/urma/urma_endpoint.cpp b/src/brpc/urma/urma_endpoint.cpp index 2545a69987..58ead7ffa1 100644 --- a/src/brpc/urma/urma_endpoint.cpp +++ b/src/brpc/urma/urma_endpoint.cpp @@ -64,6 +64,7 @@ DECLARE_int32(urma_sq_size); DECLARE_int32(urma_rq_size); DECLARE_int32(urma_cqe_poll_once); DECLARE_bool(urma_recv_zerocopy); +DECLARE_bool(urma_copy_send); DECLARE_int32(urma_zerocopy_min_size); DECLARE_int32(urma_prepared_jetty_cnt); DECLARE_bool(urma_poller_yield); @@ -686,6 +687,7 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { errno = ENOTCONN; return -1; } + const bool copy_send = FLAGS_urma_copy_send; int max_sge = GetUrmaMaxSge(); if (max_sge < 1) { max_sge = 1; @@ -716,6 +718,10 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { size_t max_len = _remote_recv_block_size > 0 ? _remote_recv_block_size : GetUrmaRecvBlockSize(); + if (copy_send) { + max_len = std::min(max_len, GetUrmaRecvBlockSize() - + (URMA_SEND_BUFFER_ALIGNMENT - 1)); + } while (sge_index < static_cast(max_sge) && this_len < max_len && current < ndata) { auto* data = reinterpret_cast(from[current]); @@ -723,8 +729,10 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { ++current; continue; } - ssize_t n = data->cut_into_sglist(sglist, &sge_index, to, - max_sge, max_len - this_len); + ssize_t n = copy_send ? + static_cast(data->cutn(to, max_len - this_len)) : + data->cut_into_sglist(sglist, &sge_index, to, + max_sge, max_len - this_len); if (n < 0) { if (errno == ERDMAMEM) { ReportFailure(UrmaFailureReason::ACCESS_ERROR); @@ -733,9 +741,18 @@ ssize_t UrmaEndpoint::CutFromIOBufList(butil::IOBuf** from, size_t ndata) { } this_len += n; } - if (sge_index == 0) { + if (this_len == 0) { break; } + if (copy_send) { + if (!CopyUrmaSendBuffer(to, &sglist[0])) { + LOG(ERROR) << "Fail to prepare aligned URMA send buffer: length=" + << this_len << " errno=" << errno; + ReportFailure(UrmaFailureReason::ACCESS_ERROR); + return -1; + } + sge_index = 1; + } urma_sg_t sg{sglist, static_cast(sge_index)}; urma_jfs_wr_t wr{}; diff --git a/src/brpc/urma/urma_helper.cpp b/src/brpc/urma/urma_helper.cpp index f7c94d4827..b603a41649 100644 --- a/src/brpc/urma/urma_helper.cpp +++ b/src/brpc/urma/urma_helper.cpp @@ -76,6 +76,8 @@ DEFINE_int32(urma_cqe_poll_once, 32, "Max completion entries polled per urma_poll_jfc call"); DEFINE_bool(urma_recv_zerocopy, true, "Use zero-copy for receives larger than --urma_zerocopy_min_size"); +DEFINE_bool(urma_copy_send, false, + "Copy host send payloads into a 64-byte aligned registered buffer"); DEFINE_int32(urma_zerocopy_min_size, 512, "Receives smaller than this many bytes are copied (not zero-copy)"); @@ -439,6 +441,45 @@ bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, return true; } +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*)) { + const size_t length = buffer->size(); + const size_t capacity = GetUrmaRecvBlockSize(); + if (length == 0 || capacity < URMA_SEND_BUFFER_ALIGNMENT || + length > capacity - (URMA_SEND_BUFFER_ALIGNMENT - 1)) { + errno = EMSGSIZE; + return false; + } + butil::IOBuf copied; + void* data = nullptr; + size_t padding = 0; + { + butil::IOBufAsZeroCopyOutputStream stream( + &copied, capacity + sizeof(butil::IOBuf::Block)); + int available = 0; + if (!stream.Next(&data, &available)) { + errno = ENOMEM; + return false; + } + padding = (URMA_SEND_BUFFER_ALIGNMENT - + reinterpret_cast(data) % URMA_SEND_BUFFER_ALIGNMENT) % + URMA_SEND_BUFFER_ALIGNMENT; + if (padding + length > static_cast(available)) { + errno = ENOMEM; + return false; + } + data = static_cast(data) + padding; + buffer->copy_to(data, length); + stream.BackUp(available - static_cast(padding + length)); + } + copied.pop_front(padding); + if (!InitUrmaLocalSge(sge, reinterpret_cast(data), length, lookup(data))) { + return false; + } + buffer->swap(copied); + return true; +} + static void GlobalRelease() { g_urma_available.store(false, butil::memory_order_release); if (g_mem_alloc_orig) { @@ -656,6 +697,7 @@ static bool GlobalUrmaInitializeImpl() { LOG(INFO) << "URMA initialized: device=" << device_name << " bonding=" << g_is_bonding_device << " max_sge=" << g_max_sge + << " copy_send=" << FLAGS_urma_copy_send << " buffer_size=" << g_pool_buffer_size << " buffer_count=" << g_pool->buffer_count(); return true; diff --git a/src/brpc/urma/urma_helper.h b/src/brpc/urma/urma_helper.h index c7439812d7..25d8d87c70 100644 --- a/src/brpc/urma/urma_helper.h +++ b/src/brpc/urma/urma_helper.h @@ -26,6 +26,10 @@ #include "bthread/types.h" #include "butil/atomicops.h" +namespace butil { +class IOBuf; +} + #if BRPC_WITH_URMA #include "urma_api.h" @@ -87,6 +91,11 @@ uint8_t GetUrmaJettyPriority(); bool InitUrmaLocalSge(urma_sge_t* sge, uint64_t address, size_t length, urma_target_seg_t* segment); +constexpr size_t URMA_SEND_BUFFER_ALIGNMENT = 64; +// Copy host payloads into one aligned block and transfer its ownership to buffer. +bool CopyUrmaSendBuffer(butil::IOBuf* buffer, urma_sge_t* sge, + urma_target_seg_t* (*lookup)(void*) = GetPoolSegFor); + // If the URMA environment is available. bool IsUrmaAvailable(); diff --git a/test/brpc_urma_copy_unittest.cpp b/test/brpc_urma_copy_unittest.cpp new file mode 100644 index 0000000000..da403f608a --- /dev/null +++ b/test/brpc_urma_copy_unittest.cpp @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include + +#if BRPC_WITH_URMA +#include "brpc/channel.h" +#include "brpc/controller.h" +#include "brpc/server.h" +#include "brpc/urma/urma_helper.h" +#include "brpc/urma/urma_recovery_policy.h" +#include "echo.pb.h" + +namespace brpc { +namespace urma { +DECLARE_bool(urma_copy_send); +DECLARE_int32(urma_buffer_count); +DECLARE_int32(urma_prepared_jetty_cnt); +DECLARE_int32(urma_max_sge); +DECLARE_int32(urma_client_handshake_version); +} // namespace urma +} // namespace brpc + +namespace { + +class CopiedEchoService : public test::EchoService { + void Echo(google::protobuf::RpcController* base, const test::EchoRequest* request, + test::EchoResponse* response, google::protobuf::Closure* done) override { + brpc::ClosureGuard guard(done); + auto* cntl = static_cast(base); + response->set_message(request->message()); + cntl->response_attachment().append(cntl->request_attachment()); + } +}; + +class UrmaCopiedRpcTest : public ::testing::TestWithParam { +protected: + static void SetUpTestSuite() { + urma_init_attr_t attr{}; + const auto status = urma_init(&attr); + if (status != URMA_SUCCESS && status != URMA_EEXIST) { + return; + } + int count = 0; + auto** devices = urma_get_device_list(&count); + _mock = devices && count > 0 && + strcmp(devices[0]->name, "mock_urma_device") == 0; + urma_free_device_list(devices); + if (status == URMA_SUCCESS) { + urma_uninit(); + } + if (!_mock) { + return; + } + brpc::urma::FLAGS_urma_copy_send = true; + brpc::urma::FLAGS_urma_buffer_count = 2048; + brpc::urma::FLAGS_urma_prepared_jetty_cnt = 0; + brpc::urma::FLAGS_urma_max_sge = 1; + brpc::urma::GlobalUrmaInitializeOrDie(); + } + void SetUp() override { + if (!_mock) { + GTEST_SKIP() << "Copied URMA RPC tests use the link-time mock"; + } + ASSERT_TRUE(brpc::urma::IsUrmaAvailable()); + ASSERT_NE(nullptr, brpc::urma::GetUrmaContext()); + brpc::urma::FLAGS_urma_client_handshake_version = GetParam(); + } + static bool _mock; +}; +bool UrmaCopiedRpcTest::_mock = false; + +TEST_P(UrmaCopiedRpcTest, copies_payload_and_attachment_across_wrs_and_credit_returns) { + CopiedEchoService service; + brpc::Server server; + ASSERT_EQ(0, server.AddService(&service, brpc::SERVER_DOESNT_OWN_SERVICE)); + brpc::ServerOptions so; + so.socket_mode = brpc::SOCKET_MODE_URMA; + ASSERT_EQ(0, server.Start("127.0.0.1:0", &so)); + brpc::Channel channel; + brpc::ChannelOptions co; + co.socket_mode = brpc::SOCKET_MODE_URMA; + co.connection_type = "single"; + co.timeout_ms = 3000; + co.max_retry = 0; + const auto before = brpc::urma::GetUrmaRecoveryStatus(); + ASSERT_EQ(0, channel.Init(server.listen_address(), &co)); + // Exceed the send window to exercise receive-credit ACKs as well as data WRs. + for (int round = 0; round < 40; ++round) { + const size_t size = round == 0 ? 0 : 20000; + const std::string payload(size, 'v'); + brpc::Controller cntl; + test::EchoRequest request; + test::EchoResponse response; + request.set_message(payload); + cntl.request_attachment().append(payload); + test::EchoService_Stub(&channel).Echo(&cntl, &request, &response, nullptr); + EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); + EXPECT_EQ(payload, response.message()); + EXPECT_EQ(payload, cntl.response_attachment().to_string()); + } + EXPECT_EQ(before.negotiated_connections + 2, + brpc::urma::GetUrmaRecoveryStatus().negotiated_connections); + server.Stop(0); + server.Join(); +} + +INSTANTIATE_TEST_SUITE_P(HandshakeVersions, UrmaCopiedRpcTest, ::testing::Values(2, 3)); + +} // namespace +#endif // BRPC_WITH_URMA + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + gflags::ParseCommandLineFlags(&argc, &argv, true); + return RUN_ALL_TESTS(); +} diff --git a/test/brpc_urma_unittest.cpp b/test/brpc_urma_unittest.cpp index da77a94d48..fb14511cfc 100644 --- a/test/brpc_urma_unittest.cpp +++ b/test/brpc_urma_unittest.cpp @@ -312,6 +312,73 @@ TEST(UrmaHelperTest, local_sge_rejects_out_of_range_and_wrong_context) { EXPECT_FALSE(urma::InitUrmaLocalSge(&sge, UINT64_MAX - 3, 2, &segment)); } +static urma_target_seg_t g_copy_test_segment; +static int g_copy_test_releases; + +static urma_target_seg_t* ResolveCopyTestSegment(void* address) { + g_copy_test_segment = urma_target_seg_t{}; + g_copy_test_segment.urma_ctx = urma::GetUrmaContext(); + g_copy_test_segment.seg.ubva.va = reinterpret_cast(address); + g_copy_test_segment.seg.len = urma::GetUrmaRecvBlockSize(); + return &g_copy_test_segment; +} + +static urma_target_seg_t* RejectCopyTestSegment(void*) { + return nullptr; +} + +static void DeleteCopyTestData(void* data) { + ++g_copy_test_releases; + delete[] static_cast(data); +} + +TEST(UrmaHelperTest, copied_send_coalesces_host_fragments_and_retains_aligned_payload) { + g_copy_test_releases = 0; + butil::IOBuf buffer; + for (const char* text : {"first", "second"}) { + const size_t length = strlen(text); + char* data = new char[length]; + memcpy(data, text, length); + ASSERT_EQ(0, buffer.append_user_data(data, length, DeleteCopyTestData)); + } + ASSERT_EQ(2u, buffer.backing_block_num()); + urma_sge_t sge{}; + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(2, g_copy_test_releases); + EXPECT_EQ("firstsecond", buffer.to_string()); + ASSERT_EQ(1u, buffer.backing_block_num()); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(reinterpret_cast(buffer.backing_block(0).data()), sge.addr); + EXPECT_EQ(buffer.size(), sge.len); + EXPECT_EQ(&g_copy_test_segment, sge.tseg); + EXPECT_EQ(nullptr, sge.user_tseg); + EXPECT_EQ("firstsecond", std::string(reinterpret_cast(sge.addr), sge.len)); + + buffer.clear(); + const std::string large(urma::GetUrmaRecvBlockSize() - + (urma::URMA_SEND_BUFFER_ALIGNMENT - 1), 'x'); + buffer.append(large); + ASSERT_TRUE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(0u, sge.addr % urma::URMA_SEND_BUFFER_ALIGNMENT); + EXPECT_EQ(large, buffer.to_string()); + EXPECT_EQ(large.size(), sge.len); +} + +TEST(UrmaHelperTest, copied_send_failure_preserves_source_payload) { + butil::IOBuf buffer; + buffer.append("unchanged"); + urma_sge_t sge{}; + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, RejectCopyTestSegment)); + EXPECT_EQ(ERDMAMEM, errno); + EXPECT_EQ("unchanged", buffer.to_string()); + const std::string large(urma::GetUrmaRecvBlockSize(), 'y'); + buffer.clear(); + buffer.append(large); + EXPECT_FALSE(urma::CopyUrmaSendBuffer(&buffer, &sge, ResolveCopyTestSegment)); + EXPECT_EQ(EMSGSIZE, errno); + EXPECT_EQ(large, buffer.to_string()); +} + // --------------------------------------------------------------------------- // SupportedByUrma: only baidu_std. // ---------------------------------------------------------------------------