feat(reconcile): manage per-pool PodDisruptionBudgets

CHANGELOG.md

@@ -46,6 +46,8 @@ All notable changes to RustFS Operator are documented in this file. The format i
   driver alongside the upstream provisioner sidecar, with dedicated ServiceAccount/RBAC.
 - Seven built-in RustFS Grafana dashboards, published by default as sidecar-discoverable
   ConfigMaps with configurable labels and namespace.
+- Per-pool PodDisruptionBudgets with `minAvailable` derived from the erasure layout RustFS
+  reports, so node drains keep write quorum; opt out with `spec.disablePodDisruptionBudget`.
 
 ### Fixed
 

crates/rustfs-admin/src/lib.rs

@@ -169,6 +169,29 @@ pub struct RustfsServerInfo {
     pub backend: Option<RustfsErasureBackend>,
     #[serde(default)]
     pub pools: Option<BTreeMap<String, BTreeMap<String, RustfsErasureSetInfo>>>,
+    #[serde(default)]
+    pub servers: Vec<RustfsServerProperties>,
+}
+
+#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)]
+pub struct RustfsServerProperties {
+    #[serde(default)]
+    pub endpoint: String,
+    #[serde(default)]
+    pub drives: Vec<RustfsDriveInfo>,
+}
+
+/// Erasure placement of one drive as reported by `/rustfs/admin/v3/info`.
+#[derive(Debug, Clone, Default, serde::Deserialize, PartialEq)]
+pub struct RustfsDriveInfo {
+    #[serde(default)]
+    pub endpoint: String,
+    #[serde(default)]
+    pub state: String,
+    #[serde(default)]
+    pub pool_index: Option<i32>,
+    #[serde(default)]
+    pub set_index: Option<i32>,
 }
 
 #[derive(Debug, Clone, serde::Deserialize, PartialEq)]

deploy/k8s-dev/operator-rbac.yaml

@@ -54,6 +54,9 @@ rules:
   - apiGroups: ["apps"]
     resources: ["statefulsets", "replicasets", "deployments"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["get", "list", "watch", "create", "patch", "delete"]
   - apiGroups: ["cert-manager.io"]
     resources: ["certificates"]
     verbs: ["get", "list", "watch", "create", "patch", "update", "delete"]

deploy/rustfs-operator/crds/tenant-crd.yaml

@@ -1561,6 +1561,16 @@ spec:
                 required:
                 - name
                 type: object
+              disablePodDisruptionBudget:
+                description: |-
+                  Disables the PodDisruptionBudgets the operator creates for each pool.
+
+                  By default every pool that can lose a server without losing write quorum gets a
+                  PodDisruptionBudget named after its StatefulSet, with `minAvailable` derived from the
+                  erasure layout reported by RustFS. Setting this to `true` deletes those budgets;
+                  PodDisruptionBudgets created outside the operator are left untouched.
+                nullable: true
+                type: boolean
               encryption:
                 description: |-
                   Encryption / KMS configuration for server-side encryption.

deploy/rustfs-operator/templates/clusterrole.yaml

@@ -56,6 +56,11 @@ rules:
     resources: ["statefulsets", "replicasets", "deployments"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 
+  # PodDisruptionBudgets for tenant pools
+  - apiGroups: ["policy"]
+    resources: ["poddisruptionbudgets"]
+    verbs: ["get", "list", "watch", "create", "patch", "delete"]
+
   # cert-manager Certificate orchestration and readiness watches
   - apiGroups: ["cert-manager.io"]
     resources: ["certificates"]

docs/operator-user-guide.md

@@ -643,6 +643,7 @@ Useful Tenant-level fields:
 | `lifecycle` | Kubernetes container lifecycle hooks. |
 | `podManagementPolicy` | StatefulSet pod management policy. |
 | `podDeletionPolicyWhenNodeIsDown` | Node-down pod deletion behavior. |
+| `disablePodDisruptionBudget` | Set to `true` to stop creating per-pool PodDisruptionBudgets and delete the ones the operator created. |
 | `securityContext` | Pod SecurityContext overrides for all RustFS Pools. |
 | `containerSecurityContext` | RustFS container SecurityContext overrides for all Pools. |
 | `hostUsers` | Optional Pod `hostUsers`. `false` isolates the user namespace (`restricted-v3`). An OpenShift empty security-context pair also defaults this to `false`. |
@@ -767,6 +768,16 @@ Multi-pool Tenants cannot contain a single-node single-disk pool. The Operator r
 
 Force deletion can have data consistency implications. Use it only when the storage backend and operational procedure are designed for that failure mode. Force deletion requires the Node object to be deleted or marked with an effective `node.kubernetes.io/out-of-service` taint that the target Pod does not tolerate, so volume detach fencing is explicit. Before using force policies, confirm the node is powered off or otherwise isolated; deleting the Node object is treated as that operational assertion.
 
+#### Pod disruption budgets
+
+The Operator creates one PodDisruptionBudget per pool, named after the pool StatefulSet and selecting the pool Pods. `minAvailable` is the pool's server count minus the number of servers that can be unavailable at once while every erasure set keeps write quorum. It is computed from the drive placement RustFS reports through the admin API:
+
+- an erasure set of `d` drives with standard parity `p` keeps write quorum while `d - p` drives are online, or `d - p + 1` when data and parity drives are equal;
+- a server that holds `k` drives of one set takes `k` drives down with it, so the set tolerates `floor(spare drives / k)` unavailable servers;
+- the smallest value across the pool's sets is used. RustFS reports a single standard parity, so the Operator uses the lower of that value and the automatic parity of each pool's set size.
+
+Pools that cannot lose a server without losing write quorum, such as single-server pools or two-server pools, get no PodDisruptionBudget. Budgets are created and updated only while every pool is rolled out, ready, and not decommissioning, and only when RustFS reports every drive of the pool online; otherwise the existing budgets are kept and the Operator retries. A node drain stops when evicting the next Pod would go below `minAvailable`. Set `spec.disablePodDisruptionBudget: true` to delete the operator-created budgets; PodDisruptionBudgets created outside the Operator are not touched. Remove any PodDisruptionBudget of your own that selects pool Pods before upgrading: Kubernetes rejects evictions of a Pod selected by more than one budget, and a budget with the same name blocks the reconcile.
+
 ### 7.5 TLS
 
 Tenant TLS is configured under `spec.tls`.

docs/operator-user-guide.zh-CN.md

@@ -603,6 +603,7 @@ ConfigMap 更新投射到 Pod。需要让更新进入运行中的 Pod 时，请
 | `lifecycle` | Kubernetes 容器 lifecycle hook。 |
 | `podManagementPolicy` | StatefulSet pod management policy。 |
 | `podDeletionPolicyWhenNodeIsDown` | 节点 NotReady/Unknown 时的 Pod 删除策略。 |
+| `disablePodDisruptionBudget` | 设为 `true` 时不再为各 Pool 创建 PodDisruptionBudget，并删除 Operator 已创建的 PodDisruptionBudget。 |
 | `securityContext` | 所有 RustFS Pool 的 Pod SecurityContext 覆盖。 |
 | `containerSecurityContext` | 所有 Pool 的 RustFS 容器 SecurityContext 覆盖。 |
 | `hostUsers` | 可选的 Pod `hostUsers`。`false` 会隔离用户命名空间（`restricted-v3`）。OpenShift 空 securityContext 对也会默认渲染为 `false`。 |
@@ -711,6 +712,16 @@ Operator 会自动管理以下环境变量：
 
 强制删除可能影响数据一致性。只有当存储后端和运维流程明确支持该故障处理方式时才应启用。强制删除要求 Node 对象已删除，或 Node 带有对目标 Pod 生效且未被该 Pod tolerate 的 `node.kubernetes.io/out-of-service` taint，确保 volume detach fencing 是显式的。启用 force 类策略前，必须先确认节点已经关机或被隔离；删除 Node 对象会被视为这一运维断言。
 
+#### Pod disruption budget
+
+Operator 为每个 Pool 创建一个 PodDisruptionBudget，名称与该 Pool 的 StatefulSet 相同，选择该 Pool 的 Pod。`minAvailable` 等于 Pool 的服务器数减去在所有纠删集保持写仲裁的前提下可以同时不可用的服务器数，依据 RustFS admin API 报告的磁盘分布计算：
+
+- 一个有 `d` 块盘、标准校验盘数为 `p` 的纠删集，在线盘数不少于 `d - p` 时保持写仲裁；数据盘与校验盘数量相等时需要 `d - p + 1`；
+- 一台服务器持有该纠删集的 `k` 块盘时，它下线会带走 `k` 块盘，因此该纠删集可容忍 `floor(冗余盘数 / k)` 台服务器不可用；
+- 取 Pool 内所有纠删集中的最小值。RustFS 只报告一个标准校验盘数，因此 Operator 取该值与每个 Pool 按纠删集大小自动计算的校验盘数中的较小值。
+
+无法在保持写仲裁的情况下失去一台服务器的 Pool（例如单服务器或双服务器 Pool）不会创建 PodDisruptionBudget。只有当所有 Pool 都完成滚动更新、全部就绪且没有进行退役，并且 RustFS 报告该 Pool 的所有磁盘在线时，才会创建或更新 PodDisruptionBudget；否则保留现有 PodDisruptionBudget 并稍后重试。节点 drain 在驱逐下一个 Pod 会低于 `minAvailable` 时停止。设置 `spec.disablePodDisruptionBudget: true` 会删除 Operator 创建的 PodDisruptionBudget；Operator 之外创建的 PodDisruptionBudget 不受影响。升级前请删除自行创建的、选择 Pool Pod 的 PodDisruptionBudget：Kubernetes 会拒绝驱逐被多个 PodDisruptionBudget 选中的 Pod，同名的 PodDisruptionBudget 会阻塞 reconcile。
+
 ### 7.5 TLS
 
 Tenant TLS 通过 `spec.tls` 配置。

src/lib.rs

@@ -29,6 +29,7 @@ use hyper_util::server::conn::auto::Builder as HyperBuilder;
 use hyper_util::service::TowerToHyperService;
 use k8s_openapi::api::apps::v1 as appsv1;
 use k8s_openapi::api::core::v1 as corev1;
+use k8s_openapi::api::policy::v1 as policyv1;
 use k8s_openapi::api::rbac::v1 as rbacv1;
 use k8s_openapi::apiextensions_apiserver::pkg::apis::apiextensions::v1::CustomResourceDefinition;
 use k8s_openapi::apimachinery::pkg::apis::meta::v1 as metav1;
@@ -290,6 +291,10 @@ async fn run_controller(
         .owns(
             Api::<appsv1::StatefulSet>::all(client.clone()),
             watcher::Config::default(),
+        )
+        .owns(
+            Api::<policyv1::PodDisruptionBudget>::all(client.clone()),
+            watcher::Config::default(),
         );
 
     let certificate_gvk = cert_manager_certificate_gvk();

src/reconcile.rs

@@ -31,6 +31,7 @@ use std::time::Duration;
 use tracing::{debug, info, warn};
 
 mod phases;
+mod pod_disruption_budget;
 mod pool_lifecycle;
 mod provisioning;
 mod tls;
@@ -113,7 +114,7 @@ pub async fn reconcile_rustfs(tenant: Arc<Tenant>, ctx: Arc<Context>) -> Result<
 
     let lifecycle_decisions = reconcile_pool_lifecycle(&ctx, &latest_tenant, &ns).await?;
 
-    let summary = reconcile_pool_statefulsets(
+    let mut summary = reconcile_pool_statefulsets(
         &ctx,
         &latest_tenant,
         &ns,
@@ -122,6 +123,14 @@ pub async fn reconcile_rustfs(tenant: Arc<Tenant>, ctx: Arc<Context>) -> Result<
         &removed_pool_cleanup,
     )
     .await?;
+    let budget_requeue = pod_disruption_budget::reconcile_pod_disruption_budgets(
+        &ctx,
+        &latest_tenant,
+        &ns,
+        summary.workloads_settled(),
+    )
+    .await?;
+    summary.set_pod_disruption_budget_requeue_after(budget_requeue);
     finalize_tenant_status(&ctx, &latest_tenant, summary, tls_plan, pod_cleanup_outcome).await
 }
 

src/reconcile/phases.rs

@@ -53,10 +53,30 @@ pub(super) struct PoolReconcileSummary {
     any_lifecycle_failed: bool,
     any_lifecycle_canceled: bool,
     lifecycle_requeue_after: Option<Duration>,
+    pod_disruption_budget_requeue_after: Option<Duration>,
     total_replicas: i32,
     ready_replicas: i32,
 }
 
+impl PoolReconcileSummary {
+    /// Every pool is rolled out and ready, and no pool lifecycle work is pending.
+    pub(super) fn workloads_settled(&self) -> bool {
+        !self.any_updating
+            && !self.any_degraded
+            && !self.any_lifecycle_reconciling
+            && !self.any_removed_pool_cleanup_reconciling
+            && !self.any_lifecycle_decommissioned
+            && !self.any_lifecycle_failed
+            && !self.any_lifecycle_canceled
+            && self.total_replicas > 0
+            && self.ready_replicas == self.total_replicas
+    }
+
+    pub(super) fn set_pod_disruption_budget_requeue_after(&mut self, after: Option<Duration>) {
+        self.pod_disruption_budget_requeue_after = after;
+    }
+}
+
 const REMOVED_POOL_CLEANUP_REQUEUE_INTERVAL: Duration = Duration::from_secs(10);
 const POD_DELETION_POLICY_REQUEUE_INTERVAL: Duration = Duration::from_secs(30);
 const SERVICE_RECREATE_REQUEUE_INTERVAL: Duration = Duration::from_secs(5);
@@ -964,8 +984,11 @@ fn reconcile_requeue_after(
     };
 
     earliest_requeue_after(
-        earliest_requeue_after(lifecycle_requeue, updating_requeue),
-        pod_cleanup_requeue,
+        earliest_requeue_after(
+            earliest_requeue_after(lifecycle_requeue, updating_requeue),
+            pod_cleanup_requeue,
+        ),
+        summary.pod_disruption_budget_requeue_after,
     )
 }