Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 5 additions & 18 deletions crates/libs/sbx/mgr/server/src/db/state_write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -181,10 +181,10 @@ edge_table! {
/// pool clone) and `transfer::reserve_import` insert a `creating` row with
/// an armed cleanup episode.
CREATE_INSERT: Request, INSERT => Creating;
/// `create_with_id`: `update_state(Created)`.
/// `create_with_id` through `write_create_completed`.
CREATE_COMPLETE: Completion, of(&[Creating]) => Created;
/// `create_with_id` in base-snapshot mode: `update_state(Suspended)`
/// after its own `created` write.
/// `create_with_id` in base-snapshot mode, through
/// `write_base_snapshot_create_completed` after its own `created` write.
CREATE_BASE_SNAPSHOT_COMPLETE: Completion, of(&[Created]) => Suspended;
/// `complete_pool_clone` and `complete_import`, under their armed
/// episodes.
Expand All @@ -200,8 +200,8 @@ edge_table! {
/// `begin_activation_with_route_evidence(Restore)` from
/// `start_from_checkpoint`.
RESTORE_BEGIN: Request, of(&[Suspended]) => Restoring;
/// `try_start_from_warm_snapshot`: `update_state(Suspended)` when the row
/// is not already `suspended`, before the cold path.
/// `try_start_from_warm_snapshot` through `write_warm_start_restaged`
/// when the row is not already `suspended`, before the cold path.
WARM_START_RESTAGE: Request, of(&[Created, Stopped]) => Suspended;
/// `insert_restoring_sandbox_row_from_snapshot` through
/// `restore_adopt_orphan_or_insert_sandbox_in_transaction`: a
Expand Down Expand Up @@ -432,19 +432,6 @@ edge_table! {
/// `resume_after_wake_lock`'s policy-free `update_state(Resuming)` after
/// a Stop completed during the pre-wake policy refresh.
RESUME_BEGIN_AFTER_STOP: KnownRace, of(&[Stopped]) => Resuming;
/// `try_start_from_warm_snapshot`'s `update_state(Suspended)`. An
/// empty-checkpoint warm Start rejects only `failed` before it, so it
/// writes from every state the generic guards admit, with no concurrency
/// needed; a row with a current activation episode answers
/// `CleanupOwned` instead.
WARM_START_UNVALIDATED: KnownRace,
StateSet::ALL
.without(Created)
.without(Stopped)
.without(Suspended)
.without(Deleting)
.without(Quarantined)
.without(Failed) => Suspended;
/// The full process-snapshot rollback's `update_state(Suspended)` after a
/// Stop admitted while it restaged the snapshot. Its own stop leaves no
/// runtime, so the second Stop is reached only in the window between
Expand Down
12 changes: 6 additions & 6 deletions crates/libs/sbx/mgr/server/src/db/tests/state_write_fixtures.rs
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ pub(crate) fn assert_unchanged(case: &str, before: &Snapshot, conn: &Connection)
}

/// Insert a plain sandbox row in `state`.
pub(super) fn insert_row(conn: &Connection, state: SandboxState) -> String {
pub(crate) fn insert_row(conn: &Connection, state: SandboxState) -> String {
let sandbox_id = SandboxId::new();
let id = sandbox_id.to_string();
insert_test_sandbox(
Expand All @@ -373,7 +373,7 @@ pub(super) fn insert_row(conn: &Connection, state: SandboxState) -> String {
/// Move a seeded row to `state` without any writer's side effects. A
/// `restoring` row carries a remote restore's three restore columns, which the
/// table's `CHECK` ties to that state.
pub(super) fn force_state(conn: &Connection, id: &str, state: SandboxState) {
pub(crate) fn force_state(conn: &Connection, id: &str, state: SandboxState) {
let restoring = state == SandboxState::Restoring;
conn.execute(
"UPDATE sandboxes
Expand All @@ -395,7 +395,7 @@ pub(super) fn restore_source() -> pplx_space_common_core::types::ExactStorageUri
}

/// Backdate every timestamp a writer may stamp, so a fresh stamp is visible.
pub(super) fn backdate(conn: &Connection, id: &str) {
pub(crate) fn backdate(conn: &Connection, id: &str) {
conn.execute(
"UPDATE sandboxes SET state_changed_at = '1', updated_at = '1' WHERE id = ?1",
params![id],
Expand Down Expand Up @@ -488,7 +488,7 @@ pub(super) fn runtime_cleared(retained: &[&str]) -> Vec<(&'static str, Cell)> {
/// A transition receipt targeting `target`, in the kind the table's `CHECK`
/// admits for it: an activation completion for created, started and paused,
/// otherwise an activation rollback.
pub(super) fn seed_receipt(conn: &Connection, id: &str, target: SandboxState) {
pub(crate) fn seed_receipt(conn: &Connection, id: &str, target: SandboxState) {
let (kind, reason) = match target {
SandboxState::Created | SandboxState::Started | SandboxState::Paused => {
("activation_completed", None)
Expand Down Expand Up @@ -528,7 +528,7 @@ pub(super) fn receipt_cleared(target: SandboxState) -> Vec<(&'static str, Cell)>
cells
}

pub(super) fn set_failure_reason(conn: &Connection, id: &str, reason: &str) {
pub(crate) fn set_failure_reason(conn: &Connection, id: &str, reason: &str) {
conn.execute(
"UPDATE sandboxes SET failure_reason = ?2 WHERE id = ?1",
params![id, reason],
Expand Down Expand Up @@ -744,7 +744,7 @@ pub(super) fn insert_creating_with(
(id, cleanup_id)
}

pub(super) fn insert_creating(
pub(crate) fn insert_creating(
conn: &Connection,
completion: CleanupCompletion,
) -> (String, SandboxCleanupEpisodeId) {
Expand Down
17 changes: 6 additions & 11 deletions crates/libs/sbx/mgr/server/src/db/tests/state_write_table.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ const ALLOWLIST: &[(SandboxState, SandboxState, Disagreement, &str)] = &[
(Creating, Created, AllowedNotRequested, COMPLETION_ONLY),
(Creating, Stopped, WriteForbidden, "import completion of a stopped export"),
(Creating, Deleting, RequestForbidden, "Delete admission supersedes the create episode"),
(Creating, Suspended, WriteForbidden, "pool clone and import completion; known race: unvalidated warm start"),
(Creating, Suspended, WriteForbidden, "pool clone and import completion"),
(Creating, CleanupRequired, WriteForbidden, "an interrupted create is handed to cleanup"),
(Creating, Quarantined, AllowedUnwritten, NO_QUARANTINE_WRITER),
(Created, Stopping, RequestForbidden, STOP_SETTLED_SOURCE),
Expand All @@ -55,14 +55,13 @@ const ALLOWLIST: &[(SandboxState, SandboxState, Disagreement, &str)] = &[
(Starting, Started, AllowedNotRequested, COMPLETION_ONLY),
(Starting, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Starting, Stopped, WriteForbidden, "cold-start rollback and startup recovery restore the entry state"),
(Starting, Suspended, WriteForbidden, "cold-start rollback restores a suspended entry state; known race: warm start"),
(Starting, Suspended, WriteForbidden, "cold-start rollback restores a suspended entry state"),
(Starting, CleanupRequired, WriteForbidden, "an interrupted cold start is handed to cleanup"),
(Starting, Quarantined, AllowedUnwritten, NO_QUARANTINE_WRITER),
(Started, Suspending, AllowedUnwritten, "suspend pauses a started row first"),
(Started, Suspended, WriteForbidden, "known race: unvalidated warm start"),
(Started, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Stopping, Stopped, AllowedNotRequested, COMPLETION_ONLY),
(Stopping, Suspended, WriteForbidden, "known races: unvalidated warm start; process-snapshot rollback in the gate-to-fence window"),
(Stopping, Suspended, WriteForbidden, "known race: process-snapshot rollback in the gate-to-fence window"),
(Stopping, CleanupRequired, WriteForbidden, "an interrupted stop is handed to cleanup"),
(Stopping, CleanupFailed, WriteForbidden, "stop cleanup failed"),
(Stopping, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
Expand All @@ -78,19 +77,15 @@ const ALLOWLIST: &[(SandboxState, SandboxState, Disagreement, &str)] = &[
(Pausing, Started, WriteForbidden, "pause rollback and startup pause recovery"),
(Pausing, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Pausing, Paused, AllowedNotRequested, COMPLETION_ONLY),
(Pausing, Suspended, WriteForbidden, "known race: unvalidated warm start"),
(Pausing, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Paused, Suspended, WriteForbidden, "known race: unvalidated warm start"),
(Paused, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Resuming, Started, AllowedNotRequested, COMPLETION_ONLY),
(Resuming, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Resuming, Paused, WriteForbidden, "resume rollback and startup resident recovery"),
(Resuming, Suspended, WriteForbidden, "known race: unvalidated warm start"),
(Resuming, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Checkpointing, Started, AllowedNotRequested, COMPLETION_ONLY),
(Checkpointing, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Checkpointing, Paused, AllowedNotRequested, COMPLETION_ONLY),
(Checkpointing, Suspended, WriteForbidden, "known race: unvalidated warm start"),
(Checkpointing, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Suspending, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Suspending, Paused, WriteForbidden, "suspend rollback and startup suspend recovery"),
Expand All @@ -106,19 +101,19 @@ const ALLOWLIST: &[(SandboxState, SandboxState, Disagreement, &str)] = &[
(Restoring, Stopping, RequestForbidden, STOP_LIVE_RUNTIME),
(Restoring, Stopped, WriteForbidden, "restore-as-new publishes a stopped disk"),
(Restoring, Paused, AllowedNotRequested, COMPLETION_ONLY),
(Restoring, Suspended, WriteForbidden, "restore rollback, cross-node publication, restore recovery; known race: warm start"),
(Restoring, Suspended, WriteForbidden, "restore rollback, cross-node publication, restore recovery"),
(Restoring, CleanupRequired, WriteForbidden, "an interrupted or ambiguous restore is handed to cleanup"),
(Restoring, Quarantined, AllowedUnwritten, NO_QUARANTINE_WRITER),
(CleanupRequired, Created, WriteForbidden, CLEANUP_SETTLES_INTENT),
(CleanupRequired, Stopping, RequestForbidden, STOP_RUNTIME_OWNER),
(CleanupRequired, Stopped, WriteForbidden, CLEANUP_SETTLES_INTENT),
(CleanupRequired, Suspended, WriteForbidden, "cleanup completion settles a mark_suspended intent; known race: warm start"),
(CleanupRequired, Suspended, WriteForbidden, "cleanup completion settles a mark_suspended intent"),
(CleanupRequired, CleanupFailed, AllowedNotRequested, "only cleanup records its own failure"),
(CleanupRequired, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(CleanupFailed, Created, WriteForbidden, CLEANUP_SETTLES_INTENT),
(CleanupFailed, Stopping, RequestForbidden, STOP_RUNTIME_OWNER),
(CleanupFailed, Stopped, WriteForbidden, CLEANUP_SETTLES_INTENT),
(CleanupFailed, Suspended, WriteForbidden, "cleanup completion settles a mark_suspended intent; known race: warm start"),
(CleanupFailed, Suspended, WriteForbidden, "cleanup completion settles a mark_suspended intent"),
(CleanupFailed, CleanupRequired, AllowedNotRequested, "only cleanup recovery re-requests cleanup"),
(CleanupFailed, Quarantined, AllowedNotRequested, RECOVERY_QUARANTINE),
(Quarantined, Started, WriteForbidden, "pause-failed quarantine recovery"),
Expand Down
20 changes: 0 additions & 20 deletions crates/libs/sbx/mgr/server/src/service/cleanup_support.rs
Original file line number Diff line number Diff line change
Expand Up @@ -238,10 +238,6 @@ pub(super) fn require_state_write(
/// error is `Internal`. Every refusal is `Aborted`: a from-state refusal
/// reports the state another transition left the row in, and a generic-guard
/// refusal or a missing row keeps `update_state`'s message.
#[cfg_attr(
not(test),
expect(dead_code, reason = "the update_state_from conversions adopt it")
)]
pub(super) fn require_state_write_from(
result: rusqlite::Result<crate::db::StateWriteOutcome>,
context: &'static str,
Expand All @@ -255,10 +251,6 @@ pub(super) fn require_state_write_from(
/// Why an `update_state_from` write did not apply. A caller that answers one
/// case differently matches it and passes the rest to
/// [`StateWriteRejected::into_status`].
#[cfg_attr(
not(test),
expect(dead_code, reason = "the update_state_from conversions adopt it")
)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum StateWriteRejected {
/// Another transition moved the row outside the edge's source states.
Expand All @@ -275,20 +267,12 @@ pub(super) enum StateWriteRejected {
}

/// The generic guards `update_state` and `update_state_from` share.
#[cfg_attr(
not(test),
expect(dead_code, reason = "the update_state_from conversions adopt it")
)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum StateWriteGuard {
Quarantined,
CleanupOwned,
}

#[cfg_attr(
not(test),
expect(dead_code, reason = "the update_state_from conversions adopt it")
)]
pub(super) fn state_write_result(
outcome: crate::db::StateWriteOutcome,
) -> Result<SandboxState, StateWriteRejected> {
Expand Down Expand Up @@ -317,10 +301,6 @@ pub(super) fn state_write_result(
}
}

#[cfg_attr(
not(test),
expect(dead_code, reason = "the update_state_from conversions adopt it")
)]
impl StateWriteRejected {
pub(super) fn into_status(self, context: &str) -> Status {
use crate::db::{SandboxStateWriteNotApplied, SandboxStateWriteOutcome};
Expand Down
14 changes: 5 additions & 9 deletions crates/libs/sbx/mgr/server/src/service/create.rs
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
use super::activation_support::{

Check warning on line 1 in crates/libs/sbx/mgr/server/src/service/create.rs

View workflow job for this annotation

GitHub Actions / rust_static / Rust clippy

[archcheck] reported by reviewdog 🐶 crates/libs/sbx/mgr/server/src/service/create.rs grew from 1195 to 1199 code lines. Once this rule is required, split the file before adding to it. Raw Output: message:"crates/libs/sbx/mgr/server/src/service/create.rs grew from 1195 to 1199 code lines. Once this rule is required, split the file before adding to it." location:{path:"crates/libs/sbx/mgr/server/src/service/create.rs" range:{start:{line:1}}} severity:WARNING source:{name:"archcheck"} code:{value:"file_size.listed_no_growth"}
ActivationFailureStage, IdentityFreeDirectActivationAuthority, complete_restore_and_mark_final,
fresh_runtime_activation_identity, runtime_activation_wire_identity_can_open_claim,
space_api_route_identity_for_runtime_activation, space_api_route_identity_from_create_request,
space_api_route_identity_to_wire, spawn_restore_state_transition,
};
use super::cache_cleanup::ensure_sandbox_cache_root;
use super::cleanup_support::{
ActivationCleanupSpec, CleanupCompletion, read_node_execution_fence, require_state_write,
};
use super::cleanup_support::{ActivationCleanupSpec, CleanupCompletion, read_node_execution_fence};
use super::create_support::*;
use super::network_runtime::*;
use super::spaced_bridge::*;
Expand Down Expand Up @@ -416,9 +414,9 @@

let update_result = {
let db = self.db_conn()?;
crate::db::update_state(&db, &id, SandboxState::Created)
write_create_completed(&db, &id)
};
if let Err(status) = require_state_write(update_result, "complete sandbox create") {
if let Err(status) = update_result {
create_cleanup.cleanup_after_failure(
SandboxQuarantineReasonCode::CreateInterrupted,
status.message(),
Expand Down Expand Up @@ -528,11 +526,9 @@
{
let update_result = {
let db = self.db_conn()?;
crate::db::update_state(&db, &id, SandboxState::Suspended)
write_base_snapshot_create_completed(&db, &id)
};
if let Err(status) =
require_state_write(update_result, "complete restored sandbox create")
{
if let Err(status) = update_result {
create_cleanup.cleanup_after_failure(
SandboxQuarantineReasonCode::CreateInterrupted,
status.message(),
Expand Down
73 changes: 73 additions & 0 deletions crates/libs/sbx/mgr/server/src/service/create_support.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use super::activation_support::space_api_route_identity_to_wire;
use super::cleanup_support::{StateWriteRejected, require_state_write_from, state_write_result};
use super::template_snapshot_support::compute_partial_sha256;
use super::validation::{
TemplateRuntimeDefaults, reject_incompatible_template_arch,
Expand Down Expand Up @@ -285,6 +286,78 @@ pub(super) fn template_create_hash(
}
}

/// Create completion's node-local SQLite `sandboxes.state` write,
/// `creating → created`. Create holds no lock between its insert and this
/// write, so a row a Stop moved to `stopping` refuses it, and the Stop keeps
/// the row.
pub(super) fn write_create_completed(db: &rusqlite::Connection, id: &str) -> Result<(), Status> {
let edge = &crate::db::edges::CREATE_COMPLETE;
require_state_write_from(
crate::db::update_state_from(db, id, edge),
"complete sandbox create",
)
}

/// Base-snapshot create's node-local SQLite `sandboxes.state` write after its
/// own `created` write, `created → suspended`, refused like
/// [`write_create_completed`].
pub(super) fn write_base_snapshot_create_completed(
db: &rusqlite::Connection,
id: &str,
) -> Result<(), Status> {
let edge = &crate::db::edges::CREATE_BASE_SNAPSHOT_COMPLETE;
require_state_write_from(
crate::db::update_state_from(db, id, edge),
"complete restored sandbox create",
)
}

fn warm_start_refused(id: &str, found: SandboxState) -> Status {
Status::failed_precondition(format!(
"cannot start sandbox {id} from a warm snapshot while local state is {found}; \
expected suspended, created or stopped"
))
}

/// Admit a warm-snapshot Start only of a row read in its node-local SQLite
/// `sandboxes.state` as a restage source or already `suspended`, before it
/// stages the base snapshot into the sandbox's suspend snapshot directory,
/// which restore treats as local suspend evidence. The restage write checks
/// again: a row can move while the snapshot is staged.
pub(super) fn admit_warm_start(sandbox: &crate::db::SandboxRow) -> Result<(), Status> {
if sandbox.state == SandboxState::Suspended
|| crate::db::edges::WARM_START_RESTAGE
.from()
.contains(sandbox.state)
{
Ok(())
} else {
Err(warm_start_refused(&sandbox.id, sandbox.state))
}
}

/// Warm-snapshot Start's node-local SQLite `sandboxes.state` write after it
/// staged the base snapshot, `created` or `stopped` → `suspended`, so the
/// checkpoint restore can claim the row. A row that moved to `suspended` in
/// the meantime needs no write. A row another transition moved anywhere else
/// is `FailedPrecondition`, as [`admit_warm_start`] answers it; a generic-guard
/// refusal or a missing row keeps `require_state_write_from`'s `Aborted`.
pub(super) fn write_warm_start_restaged(db: &rusqlite::Connection, id: &str) -> Result<(), Status> {
const CONTEXT: &str = "prepare recovered sandbox restore";
let outcome = crate::db::update_state_from(db, id, &crate::db::edges::WARM_START_RESTAGE)
.map_err(|error| Status::internal(format!("{CONTEXT}: {error}")))?;
match state_write_result(outcome) {
Ok(_)
| Err(StateWriteRejected::FromState {
found: SandboxState::Suspended,
}) => Ok(()),
Err(StateWriteRejected::FromState { found }) => Err(warm_start_refused(id, found)),
Err(rejected) => Err(rejected.into_status(CONTEXT)),
}
}

#[cfg(test)]
mod state_write_tests;
#[cfg(test)]
mod tests;

Expand Down
Loading
Loading