sbx-mgr: wake and resident resume write only over the state they expect

architecture-baseline.toml

@@ -82,7 +82,7 @@
 "crates/libs/sbx/mgr/server/src/run.rs" = 1101
 "crates/libs/sbx/mgr/server/src/sandbox_eviction.rs" = 1273
 "crates/libs/sbx/mgr/server/src/service/activation_support.rs" = 1592
-"crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 1237
+"crates/libs/sbx/mgr/server/src/service/auth_lookup.rs" = 1221
 "crates/libs/sbx/mgr/server/src/service/cache_cleanup.rs" = 1455
 "crates/libs/sbx/mgr/server/src/service/cleanup.rs" = 2704
 "crates/libs/sbx/mgr/server/src/service/create.rs" = 1195

crates/libs/sbx/mgr/server/src/db/state_write.rs

@@ -240,8 +240,8 @@ edge_table! {
     /// and the suspend auto-pause; `update_state_best_effort(Started)` after a
     /// node connect failure and in `pause_with_reconciliation`.
     PAUSE_ROLLBACK: Compensation, of(&[Pausing]) => Started;
-    /// `begin_resident_resume_with_policy`, and `resume_after_wake_lock`'s
-    /// `update_state(Resuming)` without an activation policy.
+    /// `begin_resident_resume_with_policy`, and `begin_resident_resume`'s
+    /// `update_state_from` without an activation policy.
     RESUME_BEGIN: Request, of(&[Paused]) => Resuming;
     /// `restore_resident_resume_state(Started)` through
     /// `rollback_activation_state`.
@@ -429,9 +429,6 @@ edge_table! {
     PAUSE_ROLLBACK_AFTER_STOP: KnownRace, of(&[Stopped]) => Started;
     /// Pause completion (`update_state(Paused)`) after a concurrent Stop.
     PAUSE_COMPLETE_AFTER_STOP: KnownRace, of(&[Stopped]) => Paused;
-    /// `resume_after_wake_lock`'s policy-free `update_state(Resuming)` after
-    /// a Stop completed during the pre-wake policy refresh.
-    RESUME_BEGIN_AFTER_STOP: KnownRace, of(&[Stopped]) => Resuming;
     /// The full process-snapshot rollback's `update_state(Suspended)` after a
     /// Stop admitted while it restaged the snapshot. Its own stop leaves no
     /// runtime, so the second Stop is reached only in the window between

crates/libs/sbx/mgr/server/src/db/tests/state_write_table.rs

@@ -68,7 +68,6 @@ const ALLOWLIST: &[(SandboxState, SandboxState, Disagreement, &str)] = &[
     (Stopped,         Started,         WriteForbidden,      "known race: pause rollback after Stop"),
     (Stopped,         Stopping,        RequestForbidden,    STOP_SETTLED_SOURCE),
     (Stopped,         Paused,          WriteForbidden,      "known races: pause and template-snapshot completion after Stop"),
-    (Stopped,         Resuming,        WriteForbidden,      "known race: policy-free resume after Stop"),
     (Stopped,         Suspended,       RequestForbidden,    "warm start and process-snapshot rollback restage a stopped row"),
     (Stopped,         Quarantined,     AllowedUnwritten,    NO_QUARANTINE_WRITER),
     (Deleting,        CleanupRequired, WriteForbidden,      "an interrupted delete is handed to cleanup"),

crates/libs/sbx/mgr/server/src/service/auth_lookup.rs

@@ -2,7 +2,6 @@ use super::activation_support::{
     GuestActivityKind, IdentityFreeDirectActivationAuthority, configure_process_env_for_activation,
     spawn_resume_clock_sync, wait_ready_guest_uptime_ms, wait_ready_via_spaced_host,
 };
-use super::cleanup_support::require_state_write;
 use super::create_support::*;
 use super::network_policy::EffectiveNetworkPolicy;
 use super::network_runtime::{network_policy_refresh_error_to_status, refresh_network_policy};
@@ -26,6 +25,7 @@ use pplx_space_metrics::sandbox::worker::SpaceApiRouteTokenAuthorizationResult;
 use pplx_space_net_common::{EffectiveNetworkPolicyRef, NetworkPolicy};
 use tonic::metadata::MetadataMap;
 
+mod resume_begin;
 #[cfg(test)]
 mod tests;
 
@@ -790,30 +790,8 @@ impl SandboxManagerService {
         // policy is unchanged.
         self.apply_network_policy_before_wake(sandbox_id, "resume")
             .await?;
-
-        {
-            let db = self.db_conn()?;
-            if let Some((policy, generation)) = activation_policy {
-                if !crate::db::begin_resident_resume_with_policy(
-                    &db,
-                    &sandbox.id,
-                    generation,
-                    policy,
-                )
-                .map_err(|error| {
-                    Status::internal(format!("resume policy admission failed: {error}"))
-                })? {
-                    return Err(Status::failed_precondition(
-                        "resident execution changed before resume",
-                    ));
-                }
-            } else {
-                require_state_write(
-                    crate::db::update_state(&db, &sandbox.id, SandboxState::Resuming),
-                    "begin resident resume",
-                )?;
-            }
-        }
+        self.begin_resident_resume(&sandbox.id, activation_policy)
+            .await?;
         self.watchdog.record_lifecycle_edge(&sandbox.id).await;
 
         let mut node_client = connect_node(&self.node_manager)

crates/libs/sbx/mgr/server/src/service/auth_lookup/resume_begin.rs

@@ -0,0 +1,55 @@
+//! The write that starts a resident resume: `paused → resuming` in the
+//! node-local SQLite `sandboxes.state`.
+
+use super::super::cleanup_support::require_state_write_from;
+use super::*;
+
+#[cfg(test)]
+mod tests;
+
+#[cfg(test)]
+tokio::task_local! {
+    /// Holds a resume between its pre-wake policy gate and this write, so a
+    /// test can move the row in that window.
+    static BEGIN_BARRIER: Arc<super::super::cleanup_executor::TestAdmissionBarrier>;
+}
+
+impl SandboxManagerService {
+    /// Persist `resuming` for the `paused` row `resume_after_wake_lock`
+    /// validated before its pre-wake policy gate.
+    ///
+    /// Stop takes neither the wake nor the artifact lock, and its runtime gate
+    /// admits the paused VM because its `fc.pid` resolves, so Stop can settle
+    /// `stopped` during the gate. Each branch
+    /// therefore writes only over `paused`, and a resume that lost the race
+    /// fails here, before any node-manager call: with an activation policy as
+    /// `FailedPrecondition`, without one as `Aborted`.
+    pub(super) async fn begin_resident_resume(
+        &self,
+        sandbox_id: &str,
+        activation_policy: Option<(&AutoLifecyclePolicy, Option<ExecutionGeneration>)>,
+    ) -> Result<(), Status> {
+        #[cfg(test)]
+        if let Ok(barrier) = BEGIN_BARRIER.try_with(Arc::clone) {
+            barrier.entered.notify_one();
+            let _permit = barrier.release.acquire().await.unwrap();
+        }
+        let db = self.db_conn()?;
+        if let Some((policy, generation)) = activation_policy {
+            if !crate::db::begin_resident_resume_with_policy(&db, sandbox_id, generation, policy)
+                .map_err(|error| {
+                    Status::internal(format!("resume policy admission failed: {error}"))
+                })?
+            {
+                return Err(Status::failed_precondition(
+                    "resident execution changed before resume",
+                ));
+            }
+            return Ok(());
+        }
+        require_state_write_from(
+            crate::db::update_state_from(&db, sandbox_id, &crate::db::edges::RESUME_BEGIN),
+            "begin resident resume",
+        )
+    }
+}