Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions crates/libs/sbx/mgr/server/src/db/cleanup_episodes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -477,14 +477,13 @@ pub fn request_cleanup_episode(
}
let previous_state = get_sandbox(&tx, sandbox_id)?.map(|row| row.state);
let requested_at = now_epoch_i64()?;
let preserve_restore_operation = episode.quarantine_recovery.is_some();
let changed = tx.execute(
"UPDATE sandboxes
SET state = 'cleanup_required',
failure_reason = ?1,
restore_origin = CASE WHEN ?6 THEN restore_origin ELSE NULL END,
restore_phase = CASE WHEN ?6 THEN restore_phase ELSE NULL END,
restore_storage_uri = CASE WHEN ?6 THEN restore_storage_uri ELSE NULL END,
restore_origin = NULL,
restore_phase = NULL,
restore_storage_uri = NULL,
state_changed_at = CASE
WHEN state != 'cleanup_required' THEN ?2
ELSE state_changed_at
Expand All @@ -500,7 +499,6 @@ pub fn request_cleanup_episode(
sandbox_id,
episode.completion.as_str(),
episode.keep_filesystem,
preserve_restore_operation,
],
)?;
if changed != 1 {
Expand Down Expand Up @@ -1747,6 +1745,7 @@ pub fn mark_cleanup_failed(
Ok(changed != 0)
}

#[cfg(test)]
pub fn clear_cleanup_intent(conn: &Connection, id: &str) -> Result<()> {
conn.execute(
"UPDATE sandboxes
Expand Down
1 change: 1 addition & 0 deletions crates/libs/sbx/mgr/server/src/db/quarantine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ pub(crate) fn emit_insert_quarantined_sandbox_event(
);
}

#[cfg(test)]
pub fn mark_quarantined(
conn: &Connection,
id: &str,
Expand Down
7 changes: 7 additions & 0 deletions crates/libs/sbx/mgr/server/src/db/state_transitions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,7 @@ pub(crate) fn recover_interrupted_pause_state(
/// Transfer lifecycle ownership from an armed activation to an explicit
/// stop/delete teardown. The caller is responsible for completing that
/// teardown; the activation guard becomes a superseded no-op.
#[cfg(test)]
fn update_state_superseding_activation(
conn: &Connection,
id: &str,
Expand Down Expand Up @@ -336,13 +337,15 @@ pub(crate) fn update_state_superseding_activation_in_transaction(
Ok(())
}

#[cfg(test)]
pub fn mark_stopping_superseding_activation(
conn: &Connection,
id: &str,
) -> std::result::Result<(), SupersedeActivationError> {
update_state_superseding_activation(conn, id, SandboxState::Stopping)
}

#[cfg(test)]
pub fn mark_deleting_superseding_activation(
conn: &Connection,
id: &str,
Expand Down Expand Up @@ -1086,6 +1089,7 @@ pub fn mark_suspending_quarantined(
)
}

#[cfg(test)]
pub fn mark_starting_quarantined(
conn: &Connection,
id: &str,
Expand All @@ -1103,6 +1107,7 @@ pub fn mark_starting_quarantined(
)
}

#[cfg(test)]
pub fn mark_restoring_quarantined(
conn: &Connection,
id: &str,
Expand All @@ -1120,10 +1125,12 @@ pub fn mark_restoring_quarantined(
)
}

#[cfg(test)]
pub fn rollback_starting(conn: &Connection, id: &str, state: SandboxState) -> Result<bool> {
rollback_activation_state(conn, id, SandboxState::Starting, state)
}

#[cfg(test)]
pub fn rollback_restoring(conn: &Connection, id: &str, state: SandboxState) -> Result<bool> {
rollback_activation_state(conn, id, SandboxState::Restoring, state)
}
Expand Down
85 changes: 29 additions & 56 deletions crates/libs/sbx/mgr/server/src/service/cleanup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1226,62 +1226,18 @@ impl SandboxManagerService {
.flatten()
.and_then(|state| state.parse().ok()),
);
if matches!(
match crate::db::update_state_superseding_activation_in_transaction(
&transaction,
&sandbox_id_string,
target_state,
SandboxState::Stopping | SandboxState::Deleting
) {
match crate::db::update_state_superseding_activation_in_transaction(
&transaction,
&sandbox_id_string,
target_state,
) {
Ok(()) => {}
Err(crate::db::SupersedeActivationError::QuarantineRecoveryInProgress) => {
return Err(CleanupAdmissionError::QuarantineRecoveryInProgress { sandbox_id });
}
Err(crate::db::SupersedeActivationError::StateWriteNotApplied(error))
if error.outcome == crate::db::SandboxStateWriteOutcome::Missing =>
{
if mode.auto_lifecycle_delete_expectation().is_some() {
return Err(Status::unavailable(
"sandbox disappeared during auto-lifecycle delete admission",
)
.into());
}
return Ok(CleanupAdmission::unpersisted(
sandbox_id,
current_execution_generation,
None,
fence,
));
}
Err(crate::db::SupersedeActivationError::StateWriteNotApplied(error)) => {
return Err(Status::unavailable(format!(
"cleanup lifecycle claim was superseded: {error}"
))
.into());
}
Err(error) => {
if require_atomic_admission {
return Err(Status::unavailable(format!(
"claim atomic cleanup admission failed: {error}"
))
.into());
}
return Ok(CleanupAdmission::unpersisted(
sandbox_id,
current_execution_generation,
Some(format!("claim cleanup lifecycle failed: {error}")),
fence,
));
}
Ok(()) => {}
Err(crate::db::SupersedeActivationError::QuarantineRecoveryInProgress) => {
return Err(CleanupAdmissionError::QuarantineRecoveryInProgress { sandbox_id });
}
} else {
let outcome = crate::db::update_state(&transaction, &sandbox_id_string, target_state)
.map_err(|error| {
Status::internal(format!("claim cleanup lifecycle failed: {error}"))
})?;
if outcome == crate::db::SandboxStateWriteOutcome::Missing {
Err(crate::db::SupersedeActivationError::StateWriteNotApplied(error))
if error.outcome == crate::db::SandboxStateWriteOutcome::Missing =>
{
if mode.auto_lifecycle_delete_expectation().is_some() {
return Err(Status::unavailable(
"sandbox disappeared during auto-lifecycle delete admission",
Expand All @@ -1295,9 +1251,26 @@ impl SandboxManagerService {
fence,
));
}
outcome.require_applied().map_err(|error| {
Status::unavailable(format!("cleanup lifecycle claim was superseded: {error}"))
})?;
Err(crate::db::SupersedeActivationError::StateWriteNotApplied(error)) => {
return Err(Status::unavailable(format!(
"cleanup lifecycle claim was superseded: {error}"
))
.into());
}
Err(error) => {
if require_atomic_admission {
return Err(Status::unavailable(format!(
"claim atomic cleanup admission failed: {error}"
))
.into());
}
return Ok(CleanupAdmission::unpersisted(
sandbox_id,
current_execution_generation,
Some(format!("claim cleanup lifecycle failed: {error}")),
fence,
));
}
}
if mode.auto_lifecycle_delete_expectation().is_some()
&& crate::db::cleanup_episode(&transaction, &sandbox_id_string)
Expand Down
Loading