Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions crates/l10_foundation/common/grpc/transport/src/status.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ pub const RETRY_AFTER_MS_METADATA: &str = "x-space-retry-after-ms";
/// precondition: the node cannot make the desired state converge by retrying
/// while cluster state is unchanged — the guest is unreachable because the
/// sandbox is in a non-thawing state, or the auto-lifecycle target state is
/// stuck (`Failed`/`CleanupRequired`/`CleanupFailed`/`Quarantined`). The reconcile-queue ref
/// should complete until a lifecycle or cluster repair changes the state.
/// stuck (`Failed` or `Quarantined`, or owned by a cleanup). The reconcile-queue
/// ref should complete until a lifecycle or cluster repair changes the state.
/// Absence means the failure is transient — a still-initializing guest (`guest_not_ready`),
/// a mid-transition lifecycle state (`Starting`/`Resuming`/`Suspending`) — and
/// the queue should re-drive it. The tonic `Code` alone can't carry this
Expand Down
143 changes: 122 additions & 21 deletions crates/l20_domain/sbx/mgr/protocol/src/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ pub enum LifecycleObservationError {
UnknownLabel(String),
#[error("unknown sandbox cleanup state value {0}")]
UnknownCleanupState(i32),
#[error("sandbox cleanup over lifecycle {0:?}, which no cleanup can be over")]
CleanupOutsideTeardown(SandboxLifecycleState),
}

impl SandboxLifecycleState {
Expand All @@ -35,8 +37,6 @@ impl SandboxLifecycleState {
Self::Suspending => "suspending",
Self::Suspended => "suspended",
Self::Restoring => "restoring",
Self::CleanupRequired => "cleanup_required",
Self::CleanupFailed => "cleanup_failed",
Self::Quarantined => "quarantined",
Self::Failed => "failed",
}
Expand All @@ -58,8 +58,6 @@ impl SandboxLifecycleState {
"suspending" => Ok(Self::Suspending),
"suspended" => Ok(Self::Suspended),
"restoring" => Ok(Self::Restoring),
"cleanup_required" => Ok(Self::CleanupRequired),
"cleanup_failed" => Ok(Self::CleanupFailed),
"quarantined" => Ok(Self::Quarantined),
"failed" => Ok(Self::Failed),
other => Err(LifecycleObservationError::UnknownLabel(other.to_string())),
Expand All @@ -79,41 +77,144 @@ pub fn decode_lifecycle(
}
}

/// The status a reader decides from. A node publishes a cleanup that owns the
/// row in `cleanup.state`, apart from its lifecycle, and may report the row
/// under the teardown transition it is in rather than as `CleanupRequired` or
/// `CleanupFailed`; either form decodes to the same status here, so a reader
/// answers both alike. A cleanup state this build doesn't know fails closed.
/// A cleanup that owns a sandbox row. A node publishes it in `cleanup.state`,
/// apart from the lifecycle the cleanup is over.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ObservedCleanup {
/// A forced teardown owns the row and is owed.
Required,
/// A teardown owns the row and its last attempt failed; it is retried.
Failed,
}

/// A teardown transition a cleanup can own a row over.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Teardown {
Stopping,
Deleting,
Suspending,
}

impl Teardown {
pub const fn lifecycle(self) -> SandboxLifecycleState {
match self {
Self::Stopping => SandboxLifecycleState::Stopping,
Self::Deleting => SandboxLifecycleState::Deleting,
Self::Suspending => SandboxLifecycleState::Suspending,
}
}

/// The teardown transition `lifecycle` is, if it is one.
pub const fn of(lifecycle: SandboxLifecycleState) -> Option<Self> {
match lifecycle {
SandboxLifecycleState::Stopping => Some(Self::Stopping),
SandboxLifecycleState::Deleting => Some(Self::Deleting),
SandboxLifecycleState::Suspending => Some(Self::Suspending),
_ => None,
}
}
}

/// What a reader decides from: the lifecycle a node publishes for a row no
/// cleanup owns, or the cleanup that owns it and the teardown transition it is
/// over. A status is not a lifecycle and doesn't compare equal to one: ask
/// [`Self::is_unowned`] or [`Self::unowned`] whether a row is in a lifecycle.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ObservedStatus {
/// No cleanup owns the row.
Unowned(SandboxLifecycleState),
/// `cleanup` owns the row over `teardown`.
Owned {
teardown: Teardown,
cleanup: ObservedCleanup,
},
}

impl ObservedStatus {
/// The published lifecycle: a cleanup's teardown transition while one
/// owns the row. To decide whether a row is in a lifecycle, ask
/// [`Self::is_unowned`].
pub const fn lifecycle(self) -> SandboxLifecycleState {
match self {
Self::Unowned(lifecycle) => lifecycle,
Self::Owned { teardown, .. } => teardown.lifecycle(),
}
}

/// The cleanup that owns the row, if one does.
pub const fn cleanup(self) -> Option<ObservedCleanup> {
match self {
Self::Unowned(_) => None,
Self::Owned { cleanup, .. } => Some(cleanup),
}
}

/// The lifecycle, unless a cleanup owns the row.
pub const fn unowned(self) -> Option<SandboxLifecycleState> {
match self {
Self::Unowned(lifecycle) => Some(lifecycle),
Self::Owned { .. } => None,
}
}

/// Whether the row is in `lifecycle` and no cleanup owns it.
pub fn is_unowned(self, lifecycle: SandboxLifecycleState) -> bool {
self.unowned() == Some(lifecycle)
}

/// The status label node-ctl publishes to the control plane:
/// `cleanup_required` or `cleanup_failed` for a cleanup-owned row,
/// otherwise the lifecycle's label.
pub fn label(self) -> &'static str {
match self {
Self::Unowned(lifecycle) => lifecycle.label(),
Self::Owned {
cleanup: ObservedCleanup::Required,
..
} => "cleanup_required",
Self::Owned {
cleanup: ObservedCleanup::Failed,
..
} => "cleanup_failed",
}
}
}

/// A cleanup state this build doesn't know, or a cleanup over a lifecycle no
/// cleanup can be over, fails closed.
fn decode_status(
lifecycle: Option<i32>,
cleanup: Option<&SandboxCleanup>,
) -> Result<SandboxLifecycleState, LifecycleObservationError> {
) -> Result<ObservedStatus, LifecycleObservationError> {
let lifecycle = decode_lifecycle(lifecycle)?;
let Some(raw) = cleanup.map(|cleanup| cleanup.state) else {
return Ok(lifecycle);
let raw = cleanup.map_or(SandboxCleanupState::Unspecified as i32, |cleanup| {
cleanup.state
});
let cleanup = match SandboxCleanupState::try_from(raw) {
Ok(SandboxCleanupState::Unspecified) => return Ok(ObservedStatus::Unowned(lifecycle)),
Ok(SandboxCleanupState::Required) => ObservedCleanup::Required,
Ok(SandboxCleanupState::Failed) => ObservedCleanup::Failed,
Err(_) => return Err(LifecycleObservationError::UnknownCleanupState(raw)),
};
match SandboxCleanupState::try_from(raw) {
Ok(SandboxCleanupState::Unspecified) => Ok(lifecycle),
Ok(SandboxCleanupState::Required) => Ok(SandboxLifecycleState::CleanupRequired),
Ok(SandboxCleanupState::Failed) => Ok(SandboxLifecycleState::CleanupFailed),
Err(_) => Err(LifecycleObservationError::UnknownCleanupState(raw)),
}
let teardown = Teardown::of(lifecycle)
.ok_or(LifecycleObservationError::CleanupOutsideTeardown(lifecycle))?;
Ok(ObservedStatus::Owned { teardown, cleanup })
}

impl SandboxInfo {
pub fn observed_lifecycle(&self) -> Result<SandboxLifecycleState, LifecycleObservationError> {
pub fn observed_status(&self) -> Result<ObservedStatus, LifecycleObservationError> {
decode_status(self.lifecycle, self.cleanup.as_ref())
}
}

impl SandboxListElement {
pub fn observed_lifecycle(&self) -> Result<SandboxLifecycleState, LifecycleObservationError> {
pub fn observed_status(&self) -> Result<ObservedStatus, LifecycleObservationError> {
decode_status(self.lifecycle, self.cleanup.as_ref())
}
}

impl WaitResponse {
pub fn observed_lifecycle(&self) -> Result<SandboxLifecycleState, LifecycleObservationError> {
pub fn observed_status(&self) -> Result<ObservedStatus, LifecycleObservationError> {
decode_status(self.lifecycle, self.cleanup.as_ref())
}
}
Expand Down
78 changes: 58 additions & 20 deletions crates/l20_domain/sbx/mgr/protocol/src/lifecycle/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,18 @@ use super::*;

#[test]
fn lifecycle_observation_requires_a_known_present_enum() {
let mut known = 0;
for raw in 1..=18 {
let state = SandboxLifecycleState::try_from(raw).unwrap();
let Ok(state) = SandboxLifecycleState::try_from(raw) else {
// The retired cleanup lifecycles are unknown values now.
assert!(matches!(raw, 15 | 16), "{raw}");
assert_eq!(
decode_lifecycle(Some(raw)),
Err(LifecycleObservationError::UnknownValue(raw))
);
continue;
};
known += 1;
assert_eq!(decode_lifecycle(Some(raw)).unwrap(), state);
assert_eq!(
SandboxLifecycleState::from_label(state.label()).unwrap(),
Expand All @@ -22,10 +32,14 @@ fn lifecycle_observation_requires_a_known_present_enum() {
lifecycle: Some(raw),
cleanup: None,
};
assert_eq!(info.observed_lifecycle().unwrap(), state);
assert_eq!(list.observed_lifecycle().unwrap(), state);
assert_eq!(wait.observed_lifecycle().unwrap(), state);
let unowned = ObservedStatus::Unowned(state);
assert_eq!(info.observed_status().unwrap(), unowned);
assert_eq!(list.observed_status().unwrap(), unowned);
assert_eq!(wait.observed_status().unwrap(), unowned);
assert!(unowned.is_unowned(state));
assert_eq!(unowned.label(), state.label());
}
assert_eq!(known, 16);
for raw in [0, -1, 19, i32::MAX] {
assert_eq!(
decode_lifecycle(Some(raw)),
Expand All @@ -36,30 +50,34 @@ fn lifecycle_observation_requires_a_known_present_enum() {
decode_lifecycle(None),
Err(LifecycleObservationError::Missing)
);
for label in ["cleanup_required", "cleanup_failed"] {
assert_eq!(
SandboxLifecycleState::from_label(label),
Err(LifecycleObservationError::UnknownLabel(label.to_owned()))
);
}
}

#[test]
fn an_owning_cleanup_decodes_to_its_combined_status_whatever_lifecycle_is_reported() {
fn an_owning_cleanup_is_observed_beside_its_lifecycle_and_passes_no_lifecycle_check() {
let cleanup = |state: SandboxCleanupState| SandboxCleanup {
state: state as i32,
..SandboxCleanup::default()
};
for (owner, status) in [
for (owner, observed, label) in [
(
SandboxCleanupState::Required,
SandboxLifecycleState::CleanupRequired,
ObservedCleanup::Required,
"cleanup_required",
),
(
SandboxCleanupState::Failed,
SandboxLifecycleState::CleanupFailed,
ObservedCleanup::Failed,
"cleanup_failed",
),
] {
for lifecycle in [
status,
SandboxLifecycleState::Stopping,
SandboxLifecycleState::Deleting,
SandboxLifecycleState::Suspending,
] {
for teardown in [Teardown::Stopping, Teardown::Deleting, Teardown::Suspending] {
let lifecycle = teardown.lifecycle();
let info = SandboxInfo {
lifecycle: Some(lifecycle as i32),
cleanup: Some(cleanup(owner)),
Expand All @@ -75,9 +93,17 @@ fn an_owning_cleanup_decodes_to_its_combined_status_whatever_lifecycle_is_report
lifecycle: Some(lifecycle as i32),
cleanup: Some(cleanup(owner)),
};
assert_eq!(info.observed_lifecycle().unwrap(), status, "{lifecycle:?}");
assert_eq!(list.observed_lifecycle().unwrap(), status, "{lifecycle:?}");
assert_eq!(wait.observed_lifecycle().unwrap(), status, "{lifecycle:?}");
let status = ObservedStatus::Owned {
teardown,
cleanup: observed,
};
assert_eq!(info.observed_status().unwrap(), status, "{lifecycle:?}");
assert_eq!(list.observed_status().unwrap(), status, "{lifecycle:?}");
assert_eq!(wait.observed_status().unwrap(), status, "{lifecycle:?}");
assert!(!status.is_unowned(lifecycle));
assert_eq!(status.lifecycle(), lifecycle);
assert_eq!(status.unowned(), None);
assert_eq!(status.label(), label);
}
}
// An armed activation's rollback target leaves the lifecycle as reported.
Expand All @@ -87,8 +113,8 @@ fn an_owning_cleanup_decodes_to_its_combined_status_whatever_lifecycle_is_report
..Default::default()
};
assert_eq!(
armed.observed_lifecycle().unwrap(),
SandboxLifecycleState::Starting
armed.observed_status().unwrap(),
ObservedStatus::Unowned(SandboxLifecycleState::Starting)
);
let unknown = SandboxInfo {
lifecycle: Some(SandboxLifecycleState::Stopping as i32),
Expand All @@ -99,7 +125,19 @@ fn an_owning_cleanup_decodes_to_its_combined_status_whatever_lifecycle_is_report
..Default::default()
};
assert_eq!(
unknown.observed_lifecycle(),
unknown.observed_status(),
Err(LifecycleObservationError::UnknownCleanupState(i32::MAX))
);
// A cleanup over a lifecycle no cleanup can be over names no status.
let misplaced = SandboxInfo {
lifecycle: Some(SandboxLifecycleState::Started as i32),
cleanup: Some(cleanup(SandboxCleanupState::Failed)),
..Default::default()
};
assert_eq!(
misplaced.observed_status(),
Err(LifecycleObservationError::CleanupOutsideTeardown(
SandboxLifecycleState::Started
))
);
}
3 changes: 1 addition & 2 deletions crates/l30_storage/sbx/mgr/store/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ there, but not a build or dev dependency, nor `libsqlite3-sys`.
result unknown, as under `NOT`. A write of a lifecycle state clears
`cleanup_state`, and a cleanup write sets both in one statement; the
triggers in `db/sandbox-manager/20261105142930_sandbox_cleanup_state.up.sql`
reject any other shape. The generated `legacy_state` stays only until the
next release drops it, so a rollback still finds it; don't read it.
reject any other shape.
- Only some child tables declare `ON DELETE CASCADE`: register a new sandbox
child table in `delete_sandbox_child_rows` (`src/db.rs`).
- Fixtures that sbx-mgr's other test suites share live in `db::fixture` and
Expand Down
21 changes: 4 additions & 17 deletions crates/l30_storage/sbx/mgr/store/src/db/read/counts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,11 @@
use crate::auto_lifecycle_delete_receipts::{
AutoLifecycleDeleteBlockedStats, AutoLifecycleDeleteOutboxStats,
};
use crate::db::{Connection, Result, SandboxLifecycleCount, SandboxStatusCount};
use crate::db::{Connection, Result, SandboxStatusCount};

/// Only the store defines these; callers pick one, they do not add their own.
mod sealed {
pub trait Sealed {}
impl Sealed for super::ByLifecycle {}
impl Sealed for super::ByStatus {}
impl Sealed for super::Resident {}
impl Sealed for super::MalformedSuspendedUpdatedAt {}
Expand All @@ -29,21 +28,9 @@ pub fn counts<C: Count>(conn: &Connection, count: C) -> Result<C::Output> {
count.count(conn)
}

/// Rows per combined status, the legacy lifecycle that folds a cleanup into
/// it. A state label this binary does not know fails the count rather than
/// being dropped from it.
#[derive(Clone, Copy, Debug)]
pub struct ByLifecycle;

impl Count for ByLifecycle {
type Output = Vec<SandboxLifecycleCount>;

fn count(self, conn: &Connection) -> Result<Self::Output> {
crate::db::sandbox_lifecycle_counts(conn)
}
}

/// Rows per stored lifecycle and cleanup state, each pair counted apart.
/// Rows per stored lifecycle, cleanup state and runtime, each lifecycle and
/// cleanup pair counted apart. A label this binary does not know fails the
/// count rather than being dropped from it.
#[derive(Clone, Copy, Debug)]
pub struct ByStatus;

Expand Down
Loading
Loading