diff --git a/crates/libs/sbx/mgr/server/AGENTS.md b/crates/libs/sbx/mgr/server/AGENTS.md index e418ead309..abb6e1b600 100644 --- a/crates/libs/sbx/mgr/server/AGENTS.md +++ b/crates/libs/sbx/mgr/server/AGENTS.md @@ -24,10 +24,13 @@ Follow "Privileged tests and the CI host" in `docs/rust/space-conventions.md`. (`pplx_space_node_mgr_client::connect()`, `connect_once()`, `lazy()`); give the code under test a test-served node-manager. Cancelling `shutdown_token` first protects only code paths that check it before dialing. -- `SandboxManagerServiceOptions::default()` reads `SPACE_NODE_ID` or the - host's node-id file, uses the production host-control and - credential-manager sockets, and probes `fc.pid` under `SANDBOX_RUN_DIR`. Set - every field where a test reaches them; constructors read env regardless. +- Build test services from `SandboxManagerServiceOptions::for_test()` or + `for_test_in(dir)` and override only the fields a test exercises. + `SandboxManagerServiceOptions::default()` is production wiring: it reads + `SPACE_NODE_ID` or the host's node-id file, uses the production host-control + and credential-manager sockets, probes `fc.pid` under `SANDBOX_RUN_DIR`, and + dials the host's node-manager and template-manager. Constructors read env + regardless. - Never call `db::open()`; use `db::test_conn()`. Module moves change log targets: see "Log targets" in diff --git a/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs b/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs index 0c562b8159..f28e6bca24 100644 --- a/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs +++ b/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::marker::PhantomData; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use futures_util::future::BoxFuture; @@ -131,21 +131,68 @@ pub(crate) fn fixed_client(client: NodeManagerGrpcClient) -> NodeManagerConnecto } /// A connector whose clients all dial `socket`, a test-owned path nothing -/// binds, so every node-manager call fails before reaching any server. +/// binds, so every node-manager call fails before reaching any server. As on a +/// node whose node-manager is down, `connect` and `connect_once` fail to dial +/// and a lazy client's calls fail with `Unavailable`. `connect` dials once +/// rather than waiting out the host connector's retry budget. pub(crate) fn unbound(socket: &Path) -> NodeManagerConnector { - fixed_client(lazy_client(socket)) + NodeManagerConnector::new(Arc::new(Unbound(socket.to_owned()))) +} + +/// [`unbound`] on a socket in a fresh directory that nothing creates, for +/// tests that build a subsystem context rather than a service. +pub(crate) fn unbound_in_absent_dir() -> NodeManagerConnector { + unbound( + &std::env::temp_dir() + .join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())) + .join("absent-node-manager.sock"), + ) } fn lazy_client(socket: &Path) -> NodeManagerGrpcClient { - let channel = pplx_space_common_grpc::connect_uds_channel_lazy( - socket, - tonic::transport::Endpoint::from_static("http://[::]:50051"), - ); + let channel = pplx_space_common_grpc::connect_uds_channel_lazy(socket, endpoint()); + client(channel) +} + +fn endpoint() -> tonic::transport::Endpoint { + tonic::transport::Endpoint::from_static("http://[::]:50051") +} + +fn client(channel: tonic::transport::Channel) -> NodeManagerGrpcClient { NodeManagerGrpcClient::new(pplx_space_common_svc::bootstrap::wrap_grpc_client_channel( channel, )) } +struct Unbound(PathBuf); + +impl Unbound { + async fn dial(&self) -> Result { + let channel = pplx_space_common_grpc::connect_uds_channel_with_endpoint( + "node-manager", + &self.0, + endpoint(), + ) + .await?; + Ok(client(channel)) + } +} + +#[async_trait::async_trait] +impl NodeManagerTransport for Unbound { + async fn connect(&self) -> Result { + self.dial().await + } + + async fn connect_once(&self) -> Result { + self.dial().await + } + + fn lazy(&self) -> NodeManagerGrpcClient { + lazy_client(&self.0) + } +} + struct InProcess(NodeManagerGrpcClient); #[async_trait::async_trait] diff --git a/crates/libs/sbx/mgr/server/src/node_state_sync.rs b/crates/libs/sbx/mgr/server/src/node_state_sync.rs index 29cdc680d4..eb7c939634 100644 --- a/crates/libs/sbx/mgr/server/src/node_state_sync.rs +++ b/crates/libs/sbx/mgr/server/src/node_state_sync.rs @@ -15,7 +15,7 @@ use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; use tonic::Request; -const SKIP_NODE_PUBLISH_METADATA: &str = "x-space-skip-cluster-sync"; +pub(crate) const SKIP_NODE_PUBLISH_METADATA: &str = "x-space-skip-cluster-sync"; const DIRTY_MARK_INITIAL_RETRY: std::time::Duration = std::time::Duration::from_secs(1); const DIRTY_MARK_MAX_RETRY: std::time::Duration = std::time::Duration::from_secs(300); const DIRTY_OUTBOX_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); diff --git a/crates/libs/sbx/mgr/server/src/run/tests.rs b/crates/libs/sbx/mgr/server/src/run/tests.rs index 7d1e882bf4..0446d94f0f 100644 --- a/crates/libs/sbx/mgr/server/src/run/tests.rs +++ b/crates/libs/sbx/mgr/server/src/run/tests.rs @@ -145,7 +145,7 @@ async fn recovery_wait_does_not_hold_the_dependency_work_gate() { for path in [&capability, &admission, &work] { std::fs::File::create(path).unwrap(); } - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); service .enable_dependency_restart_drain(&capability, &admission, &work) .unwrap(); @@ -774,7 +774,7 @@ async fn grpc_writer_during_pre_handoff_drain_returns_retry_on_successor() { let listener = pplx_space_common_svc::bind_unix_socket(&socket_path, UnixSocketOptions::new(0o600)) .unwrap(); - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let rpc_drain_token = CancellationToken::new(); let shutdown_token = CancellationToken::new(); let server_shutdown = shutdown_token.clone(); @@ -821,7 +821,7 @@ async fn grpc_in_flight_request_interrupted_at_handoff_is_not_marked_replayable( let listener = pplx_space_common_svc::bind_unix_socket(&socket_path, UnixSocketOptions::new(0o600)) .unwrap(); - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let rpc_drain_token = CancellationToken::new(); let shutdown_token = CancellationToken::new(); let server_shutdown = shutdown_token.clone(); diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs index e78a611cc0..675caa8de2 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs @@ -841,7 +841,7 @@ async fn evict_tick_below_engage_returns_before_candidate_scan() { [], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.50, @@ -892,7 +892,7 @@ fn resident_rows_count_every_state() { async fn evict_tick_without_disk_usage_fails_without_scanning() { let conn = open_test_db(); insert_sandbox(&conn, "sbx-cold", "suspended", 1_000); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.0, @@ -924,7 +924,7 @@ async fn evict_tick_does_not_connect_when_all_candidates_are_backed_off() { let conn = open_test_db(); insert_sandbox(&conn, "sbx-backed-off", "suspended", 1_000); insert_pinned_artifact(&conn, "sbx-backed-off", &v7(1_000).to_string(), Some("123")); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.50, @@ -1308,7 +1308,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( use crate::state::SandboxState; use pplx_space_common_core::types::SandboxId; - let service = SandboxManagerService::new(open_test_db()); + let service = SandboxManagerService::for_test(open_test_db()); let unknown = service .reupload_legacy_sandbox_for_eviction(&SandboxId::new().to_string()) .await @@ -1329,7 +1329,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( let conn = open_test_db(); let id = SandboxId::new().to_string(); insert_sandbox(&conn, &id, state.as_str(), 0); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let refused = service .reupload_legacy_sandbox_for_eviction(&id) @@ -1350,7 +1350,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( insert_sandbox(&conn, &id, SandboxState::Restoring.as_str(), 0); let service = SandboxManagerService::new_with_config( conn, - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { hugepages: true, ..Default::default() @@ -1370,7 +1370,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( /// Empty cluster-api wiring is operator-fixable, so eviction retries the /// reupload instead of skipping the sandbox. async fn assert_missing_wiring_reupload_is_retried() { - let service = SandboxManagerService::new(open_test_db()); + let service = SandboxManagerService::for_test(open_test_db()); let refused = service .reupload_legacy_sandbox_for_eviction( &pplx_space_common_core::types::SandboxId::new().to_string(), diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs b/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs index 37d232b255..4ba5c856c6 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs @@ -590,7 +590,7 @@ async fn restore_finalizer_retains_cleanup_until_state_persistence_succeeds() { SandboxCleanupEpisodeId::new(), ) }; - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let temp = tempfile::tempdir().unwrap(); let snapshot_dir = temp.path().join("snapshot"); std::fs::create_dir(&snapshot_dir).unwrap(); @@ -631,7 +631,7 @@ async fn restore_finalizer_retains_cleanup_until_state_persistence_succeeds() { #[tokio::test] async fn cold_finalizer_retains_cleanup_until_state_persistence_succeeds() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let (cleanup_tx, cleanup_rx) = tokio::sync::oneshot::channel(); let cleanup_guard = ActivationCleanupGuard::with_interruption_reason( SandboxRuntimeLabel::parse("runtime1").unwrap(), @@ -655,7 +655,7 @@ async fn cold_finalizer_retains_cleanup_until_state_persistence_succeeds() { #[tokio::test] async fn cold_finalizer_disarms_cleanup_after_state_persistence_succeeds() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let (cleanup_tx, cleanup_rx) = tokio::sync::oneshot::channel(); let cleanup_guard = ActivationCleanupGuard::with_interruption_reason( SandboxRuntimeLabel::parse("runtime1").unwrap(), diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs b/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs index f6c12d393c..5bddc2a23f 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs @@ -200,7 +200,7 @@ async fn exercise_recovery(restore: bool) { SandboxManagerServiceOptions { credential_manager_socket_path: node_socket, shutdown_token: shutdown.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let _gateway = super::super::super::gateway_identity::tests::GatewayNode::attach(&mut service); diff --git a/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs b/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs index 9f1e758312..8fe84a6f9f 100644 --- a/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs @@ -4,7 +4,7 @@ use crate::service::activation_support::internal_request; fn test_service() -> SandboxManagerService { let connection = crate::db::test_conn(); - SandboxManagerService::new(connection) + SandboxManagerService::for_test(connection) } fn expect_authorization_error(result: Result) -> Status { diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs index 13e272ece4..210e7f8413 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs @@ -21,7 +21,7 @@ async fn gateway_identity_mapping_preserves_exact_pid_and_lifecycle_authority() crate::db::update_netgtw_pid(&conn, &id, owner, 4242).unwrap(); let service = SandboxManagerService::new_with_options( conn, - super::super::SandboxManagerServiceOptions::default(), + super::super::SandboxManagerServiceOptions::for_test(), ); for (pid, uid, allowed) in [ (4242, owner, true), diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs index 4ee88fb043..095d9de720 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs @@ -7,7 +7,7 @@ use pplx_space_common_svc::space_api_route::{ }; fn service() -> SandboxManagerService { - SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()) + SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()) } fn node_api_peer() -> AuthorizedUdsPeer { diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs index a6db53a4d0..ddad9fc8a0 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs @@ -73,7 +73,7 @@ fn fixture_with_claims( rusqlite::Connection::open_in_memory().unwrap(), SandboxManagerServiceOptions { route_ed25519_verification_bundle: Some(Arc::new(bundle)), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let mut request = tonic::Request::new(()); diff --git a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs index a218e7bb59..1d90aaf1b9 100644 --- a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs @@ -16,7 +16,7 @@ async fn periodic_inventory_progresses_in_pages_without_treating_other_pages_as_ } let stale = SandboxId::new(); insert_test_sandbox(&conn, &stale.to_string(), SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut scan = PeriodicInventoryScan::default(); let shutdown = CancellationToken::new(); let paths = SandboxInventoryPaths::new(cache.path(), runtime.path()); @@ -104,7 +104,7 @@ async fn periodic_inventory_names_only_rowless_cache_directories_for_inspection( std::fs::create_dir(cache.path().join(id.to_string())).unwrap(); } insert_test_sandbox(&conn, &owned.to_string(), SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let batch = service .discover_periodic_inventory_in( &mut PeriodicInventoryScan::default(), @@ -190,7 +190,7 @@ async fn orphan_quarantine_survives_inventory_once_its_runtime_is_gone() { Some(SandboxQuarantineReasonCode::OrphanedSandboxRowMissing.as_str()) ); std::fs::remove_dir_all(&sandbox_run_dir).unwrap(); - let service = crate::service::tests::service_tests::service_with_runtime_liveness(conn, false); + let service = crate::service::tests::service_tests::service_with_runtime_probe(conn, false); let before = database_rows(&service.db_conn().unwrap()); reconcile_one_periodic_page(&service, paths); @@ -267,8 +267,7 @@ async fn startup_and_periodic_inventory_remove_only_rows_whose_state_requires_lo .collect::>(); assert!(!expected_removed.is_empty()); - let service = - crate::service::tests::service_tests::service_with_runtime_liveness(conn, false); + let service = crate::service::tests::service_tests::service_with_runtime_probe(conn, false); let removed_or_published = if periodic { reconcile_one_periodic_page(&service, paths); crate::db::list_node_dirty_refs(&service.db_conn().unwrap(), 1024) diff --git a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs index c585f66b62..3022d3afc9 100644 --- a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs @@ -398,7 +398,7 @@ async fn row_cleanup_retires_policy_only_after_identity_is_deleted() { .unwrap(); episode.id }); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let queue = &service.network_policy_reconcile_queue; queue.enqueue(id.clone()); let old_attempt = queue.take_pending(&Default::default(), 1).pop().unwrap(); @@ -455,7 +455,7 @@ async fn inventory_retires_registries_atomically_and_retries_busy_registrations( let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cache = tempfile::tempdir().unwrap(); let run = tempfile::tempdir().unwrap(); service.network_policy_reconcile_queue.enqueue(id.clone()); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs index c5d09a03c3..47c8bd90a6 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs @@ -42,8 +42,6 @@ pub(super) struct CleanupExecutor { #[cfg(test)] waiter_read_barrier: Mutex>>, #[cfg(test)] - pub(super) run_dir: Mutex>, - #[cfg(test)] pub(super) cache_dir: Mutex>, wake: tokio::sync::Notify, pub(super) active: dashmap::DashMap, @@ -69,8 +67,6 @@ impl Default for CleanupExecutor { #[cfg(test)] waiter_read_barrier: Mutex::new(None), #[cfg(test)] - run_dir: Mutex::new(None), - #[cfg(test)] cache_dir: Mutex::new(None), wake: tokio::sync::Notify::new(), active: dashmap::DashMap::new(), diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs index ae71d8450e..09b0f834b1 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs @@ -50,11 +50,9 @@ async fn blocking_stop_follows_a_retryable_attempt_to_durable_completion() { let sandbox_id = SandboxId::new(); let subject = sandbox_id.to_string(); insert_test_sandbox(&conn, &subject, SandboxState::Stopping); - let sockets = tempfile::tempdir().unwrap(); let service = crate::service::tests::node_manager_transport_tests::service_with_node_manager( conn, node_manager.connector(), - sockets.path(), ); let execution_fence = crate::db::node_execution_fence(&service.db.lock().unwrap(), &subject).unwrap(); @@ -565,14 +563,12 @@ async fn lifecycle_and_job_completion_commit_together_and_recover( ); // Virtual time from here on. A case that registered an activation episode // also registered the sandbox with the snapshot scheduler, whose first tick - // opens a node-manager connection; that connect does not watch shutdown, so - // the drain below has to wait out its whole `tokio::time` retry budget - // (~9.5 s against an absent socket) before the task can observe the - // cancellation. Pausing collapses the backoff without changing what the - // drain observes: the connects still run and still fail, and every - // assertion after this point reads state the drain is required to have - // settled. Only the wait is virtual, so everything above runs on the real - // clock exactly as the executor's own timers need. + // opens a node-manager connection; that connect does not watch shutdown. + // The test service's node-manager is an absent socket dialed once, so the + // connect fails at once, and pausing only keeps any later scheduler timer + // from holding the drain on the real clock. Every assertion after this + // point reads state the drain is required to have settled. Everything + // above runs on the real clock exactly as the executor's own timers need. tokio::time::pause(); service.shutdown_token.cancel(); service.task_tracker.close(); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs index 78a89b0152..8de30f8770 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs @@ -206,10 +206,7 @@ impl SandboxManagerService { .map_err(|_| Status::unavailable("cleanup storage is shutting down"))?; #[cfg(test)] let run_dir = self - .cleanup_executor - .run_dir - .lock() - .unwrap() + .cleanup_run_dir .clone() .unwrap_or_else(|| PathBuf::from(RUN_DIR)); let cache_dir = self.sandbox_cache_root(); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs index 854419a817..42dc627dc0 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs @@ -6,7 +6,7 @@ use crate::service::tests::service_tests::insert_test_sandbox; async fn inventory_stream_cancels_while_waiting_for_storage_capacity() { use tokio_stream::StreamExt; for reason in ["deadline", "shutdown", "reader"] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let stream_capacity = service.cleanup_target_streams.available_permits(); let cache = tempfile::tempdir().unwrap(); let permits = service.cleanup_io.available_permits(); @@ -159,7 +159,7 @@ async fn streamed_inventory_includes_rowless_targets_and_preserves_bad_metadata( [&malformed], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cache = tempfile::tempdir().unwrap(); for id in [&persisted, &rowless] { std::fs::create_dir(cache.path().join(id)).unwrap(); @@ -191,7 +191,7 @@ async fn streamed_inventory_includes_rowless_targets_and_preserves_bad_metadata( #[tokio::test] async fn inventory_stream_owns_capacity_and_reports_failure_instead_of_clean_eof() { use tokio_stream::StreamExt; - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let cache = tempfile::tempdir().unwrap(); let deadline = tokio::time::Instant::now() + Duration::from_secs(120); let capacity = service.cleanup_target_streams.available_permits(); @@ -217,7 +217,7 @@ async fn inventory_stream_owns_capacity_and_reports_failure_instead_of_clean_eof assert_eq!(service.cleanup_target_streams.available_permits(), capacity); for failure in ["deadline", "database"] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let deadline = if failure == "deadline" { tokio::time::Instant::now() } else { diff --git a/crates/libs/sbx/mgr/server/src/service/create/tests.rs b/crates/libs/sbx/mgr/server/src/service/create/tests.rs index 0927a36dbd..0bb09e14a2 100644 --- a/crates/libs/sbx/mgr/server/src/service/create/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/create/tests.rs @@ -57,7 +57,7 @@ async fn create_refusals_before_insert_persist_nothing() { crate::db::test_conn(), SandboxManagerServiceOptions { local_node_id: Some(NodeId::new()), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let id = SandboxId::new().to_string(); diff --git a/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs b/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs index 425243bf02..44fe7b1b3a 100644 --- a/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs @@ -351,7 +351,7 @@ async fn create_intent_preserves_immutables_independently_of_pool_occupancy() { ) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for has_snapshot in [false, true] { let snapshot_dir = has_snapshot.then(|| snapshot.path().to_str().unwrap()); for (field, request, incompatible_with_pool) in &cases { diff --git a/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs b/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs index 0f2a744525..21a67afdd5 100644 --- a/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs @@ -12,7 +12,7 @@ struct Fixture { impl Fixture { fn new() -> Self { - Self::with_options(SandboxManagerServiceOptions::default()) + Self::with_options(SandboxManagerServiceOptions::for_test()) } fn with_options(options: SandboxManagerServiceOptions) -> Self { @@ -202,7 +202,7 @@ async fn gateway_authority_uses_the_configured_node_socket() { let listener = tokio::net::UnixListener::bind(&socket).unwrap(); let fixture = Fixture::with_options(SandboxManagerServiceOptions { credential_manager_socket_path: socket, - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }); let expected = fixture.request.csr.clone(); let issue = tower::service_fn(move |request: Request| { diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs index 9e28442e0e..7a0a96d8e0 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs @@ -118,20 +118,20 @@ async fn create_checks_auto_lifecycle_authority_before_template_lookup() { for case in cases { let templates = Arc::new((case.templates)()); let node_manager = FakeNodeManager::default().serve(); - let sockets = tempfile::tempdir().unwrap(); - let service = crate::service::tests::node_manager_transport_tests::isolated_service( + let service = SandboxManagerService::new_with_options( crate::db::test_conn(), - node_manager.connector(), - Arc::clone(&templates) as Arc, - sockets.path(), + SandboxManagerServiceOptions { + node_manager: node_manager.connector(), + template_resolver: Arc::clone(&templates) as _, + ..SandboxManagerServiceOptions::for_test() + }, ); - let sandbox_id = SandboxId::new().to_string(); let mut request = Request::new(CreateRequest { template: "seed".into(), owner: "root@local".into(), runtime: pplx_space_common_core::types::RUNTIME_FIRECRACKER_ALIAS.into(), boot_mode: BootMode::Cold as i32, - sandbox_id: Some(sandbox_id.clone()), + sandbox_id: Some(SandboxId::new().to_string()), auto_lifecycle: Some(Default::default()), auto_lifecycle_authority: case.authority, ..Default::default() @@ -169,10 +169,5 @@ async fn create_checks_auto_lifecycle_authority_before_template_lookup() { assert!(lookup.metadata.is_empty(), "{}", case.name); } assert!(node_manager.calls().is_empty(), "{}", case.name); - assert!( - !crate::db::sandbox_exists(&service.db_conn().unwrap(), &sandbox_id).unwrap(), - "{}", - case.name - ); } } diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs index c8d7ea8eb1..595061be7c 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs @@ -30,48 +30,27 @@ fn enter_isolated_test_process() -> bool { ]) } -/// A service built from explicit options. `Default` options read the node -/// identity from a fixed host file and probe runtimes under the fixed run -/// directory; here the node identity is fixed, runtime liveness reads -/// `scratch`, the host-control, credential-manager and node-manager sockets -/// are absent paths under `scratch`, and every template lookup is refused. +/// A service whose host dependencies all resolve under `scratch`; see +/// [`SandboxManagerServiceOptions::for_test_in`]. fn isolated_service( conn: rusqlite::Connection, scratch: &std::path::Path, node_role: Option, ) -> SandboxManagerService { - let run_dir = scratch.to_owned(); SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: scratch.join("absent-host-control.sock"), - credential_manager_socket_path: scratch.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(move |id| { - crate::service::cache_cleanup::is_sandbox_running_in(&run_dir, id) - }), - shutdown_token: tokio_util::sync::CancellationToken::new(), - task_tracker: tokio_util::task::TaskTracker::new(), - local_node_id: Some(pplx_space_common_core::types::NodeId::from_uuid( - uuid::Uuid::from_u128(0x019f_8000_0000_7000_8000_0000_0000_00fe), - )), node_role, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::fake::unbound( - &scratch.join("absent-node-manager.sock"), - ), - template_resolver: Arc::new( - crate::template_resolver::fake::FakeTemplateResolver::refusing( - Status::unavailable("template-manager is outside this test"), - ), - ), + ..SandboxManagerServiceOptions::for_test_in(scratch) }, ) } /// An auto-lifecycle authority no proto value names. Create parses it last -/// before template lookup, which would contact the fixed template-manager -/// socket. +/// before template lookup. A lookup past the barrier stops at the test +/// service's template-manager fake, whose socket is unbound: the lookup fails +/// to connect (`TemplateLookupError::Connect`), which create reports as +/// `Internal` "cannot connect to template-manager". const BARRIER_AUTHORITY: i32 = 98; /// The barrier's refusal; no case expects this message for its own refusal. const BARRIER_REFUSAL: &str = "policy authority proto value 98 is unknown"; @@ -290,7 +269,6 @@ async fn forwarded_create_refuses_owner_and_network_policy_authority_before_temp network_policy_authority: case.network_policy_authority.unwrap_or_default(), ..create_refused_before_template_lookup() }; - let sandbox_id = request.sandbox_id.clone().unwrap(); let error = service .create(peer_request( @@ -301,10 +279,6 @@ async fn forwarded_create_refuses_owner_and_network_policy_authority_before_temp .expect_err(case.name); assert_eq!(error.code(), case.code, "{context}: {error}"); assert!(error.message().contains(case.message), "{context}: {error}"); - assert!( - !crate::db::sandbox_exists(&service.db_conn().unwrap(), &sandbox_id).unwrap(), - "{context}" - ); } } } @@ -345,7 +319,6 @@ impl CleanupFixture { ) -> Self { let run_dir = tempfile::tempdir().unwrap(); let service = isolated_service(conn, run_dir.path(), node_role); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let dispatch_permits = Arc::new(tokio::sync::Semaphore::new(dispatch_permits)); let permits = Arc::clone(&dispatch_permits); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new(move |_, _| { @@ -685,7 +658,7 @@ async fn nonblocking_cleanup_returns_once_admitted_and_honors_skip_node_publish( let mut request = peer_request((), Some(root_peer())); if skip_node_publish { request.metadata_mut().insert( - "x-space-skip-cluster-sync", + crate::node_state_sync::SKIP_NODE_PUBLISH_METADATA, tonic::metadata::MetadataValue::from_static("true"), ); } @@ -752,7 +725,7 @@ async fn nonblocking_cleanup_returns_once_admitted_and_honors_skip_node_publish( // completion marks the sandbox dirty just after the job settles. within_cleanup_wait(&context, async { while !fixture.dirty_ids().contains(&id) { - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(5)).await; } }) .await; @@ -872,7 +845,9 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy uppercase_id: bool, template_filter: Option<&'static str>, expected_policy: &'a AutoLifecyclePolicy, - refusal: Option, + /// The refusal's code and part of its message; one code comes from + /// more than one branch. + refusal: Option<(tonic::Code, &'static str)>, } for case in [ Case { @@ -880,21 +855,27 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy uppercase_id: false, template_filter: Some("seed"), expected_policy: &policy, - refusal: Some(tonic::Code::Internal), + refusal: Some(( + tonic::Code::Internal, + "auto-lifecycle delete requires one exact sandbox id", + )), }, Case { name: "non-canonical id", uppercase_id: true, template_filter: None, expected_policy: &policy, - refusal: Some(tonic::Code::Internal), + refusal: Some(( + tonic::Code::Internal, + "auto-lifecycle delete requires a canonical sandbox id", + )), }, Case { name: "stale policy", uppercase_id: false, template_filter: None, expected_policy: &stale, - refusal: Some(tonic::Code::Aborted), + refusal: Some((tonic::Code::Aborted, "auto-lifecycle delete lost its")), }, Case { name: "current policy", @@ -950,9 +931,10 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy let result = within_cleanup_wait(case.name, delete).await; match case.refusal { - Some(code) => { + Some((code, message)) => { let error = result.unwrap_err(); - assert_eq!(error.code(), code, "{}", case.name); + assert_eq!(error.code(), code, "{}: {error}", case.name); + assert!(error.message().contains(message), "{}: {error}", case.name); // The reconciler retries a lost policy comparison by this key. assert_eq!( error.metadata().contains_key( @@ -1050,7 +1032,9 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: bool, hold_cleanup_fence: bool, blocking: bool, - refusal: Option, + /// The refusal's code and part of its message; one code comes from + /// more than one branch. + refusal: Option<(tonic::Code, &'static str)>, } for case in [ Case { @@ -1061,7 +1045,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "pool clone publication still owns this sandbox", + )), }, Case { name: "template manager after quarantine", @@ -1071,7 +1058,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "clone disposal hold requires an explicit operator delete", + )), }, Case { name: "auto-lifecycle after quarantine", @@ -1081,7 +1071,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: true, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "clone disposal hold requires an explicit operator delete", + )), }, Case { name: "operator while cleanup is busy", @@ -1091,7 +1084,7 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: true, blocking: false, - refusal: Some(tonic::Code::Unavailable), + refusal: Some((tonic::Code::Unavailable, "clone disposal is busy")), }, Case { name: "operator after quarantine", @@ -1170,10 +1163,12 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() let episode = crate::db::cleanup_episode(&fixture.service.db_conn().unwrap(), &id).unwrap(); match case.refusal { - Some(code) => { + Some((code, message)) => { let episode = episode.unwrap(); assert_eq!(episode.id, cleanup_id, "{}", case.name); - assert_eq!(result.unwrap_err().code(), code, "{}", case.name); + let error = result.unwrap_err(); + assert_eq!(error.code(), code, "{}: {error}", case.name); + assert!(error.message().contains(message), "{}: {error}", case.name); assert_eq!(fixture.state(&id), Some(held_state), "{}", case.name); assert_eq!(episode.requested_at, None, "{}", case.name); assert!(fixture.active_cleanup_job(&id).is_none(), "{}", case.name); diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs index 1406d5f6d0..a22d15935f 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs @@ -247,7 +247,7 @@ async fn artifact_backup_admission_refuses_sources_it_cannot_back_up() { ); } } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let requested_id = if case == "blank id" { " " } else { &id }; let error = service @@ -272,7 +272,7 @@ async fn artifact_backup_admission_refuses_sources_it_cannot_back_up() { /// marker in the backup refusal. Eviction's reading of the matching reupload /// refusal is pinned against its classifier in `crate::sandbox_eviction`. async fn assert_refused_as_missing_wiring() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let unknown = SandboxId::new().to_string(); let enqueue = service .enqueue_artifact_backup(artifact_backup_request( @@ -425,7 +425,7 @@ async fn durable_artifact_backup_persists_the_submitted_artifact_identity() { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Suspended); - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.snapshot_upload_config = case.default_bucket.map(|b| Arc::new(upload_config(b))); let sandbox = service.lookup_sandbox(&id).unwrap(); let root = tempfile::tempdir().unwrap(); @@ -663,7 +663,7 @@ async fn service_with_template_policy( egress: Egress, generation: u64, ) -> SandboxManagerService { - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = EffectiveNetworkPolicy::new( stored_policy(egress), generation, @@ -874,7 +874,7 @@ async fn set_network_policy_rejects_malformed_requests_without_mutation() { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for (name, request, expected) in [ ( "unknown sandbox", @@ -1083,7 +1083,7 @@ async fn kill_refuses_signals_it_cannot_deliver_before_cleanup_admission() { if let Some(state) = state { insert_test_sandbox(&conn, &id, state); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.begin_draining_for_restart(); (service, id) }; diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs index ec457aac13..2481a25170 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs @@ -113,7 +113,7 @@ async fn teardown_reuses_ready_backup_but_waits_for_active_local_upload() { ) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let pending = service.ensure_teardown_suspend_backup( &row, super::TeardownSuspendBackup { @@ -178,7 +178,7 @@ async fn failed_teardown_defers_repeated_polls_without_holding_driver_capacity() let sandbox = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); insert_stranded_teardown_suspend_sandbox(&conn, sandbox, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .drive_teardown_suspend(sandbox, generation, SandboxEventTrigger::System) .await diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs index 82ba947137..05109afe47 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs @@ -37,7 +37,7 @@ async fn set_network_policy_rejects_local_authority_from_forwarded_service_peer( let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let allow_all = pplx_space_net_common::NetworkPolicyV1::allow_all(); let make_request = |uid: u32, gid: u32| { @@ -108,7 +108,7 @@ async fn set_auto_lifecycle_policy_rejects_local_authority_from_forwarded_servic let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let make_request = |uid: u32, gid: u32| { let mut request = tonic::Request::new(SetAutoLifecyclePolicyRequest { @@ -180,7 +180,7 @@ async fn template_notification_matches_inherited_sandboxes_of_both_authorities() .unwrap(); } } - let service = crate::service::SandboxManagerService::new(conn); + let service = crate::service::SandboxManagerService::for_test(conn); let response = service .template_network_policy_updated(internal_request( crate::proto::TemplateNetworkPolicyUpdatedRequest { @@ -715,7 +715,7 @@ fn current_instance_handler_refuses_incomplete_suspend_without_publishing_a_pin( conn.execute("UPDATE sandboxes SET state = 'restoring' WHERE id = ?1", [&id]) .unwrap(); } - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.config.hugepages = case == "unsupported"; let error = service .enqueue_artifact_backup(crate::service::activation_support::internal_request( @@ -962,7 +962,7 @@ async fn suspended_artifact_adoption_race_uses_existing_artifact_id() { assert!( crate::db::adopt_suspended_local_artifact(&conn, sandbox_id, first, snapshot_dir).unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let resolved = service .adopt_or_get_suspended_local_artifact(sandbox_id, second, snapshot_dir) @@ -1116,7 +1116,7 @@ async fn permanently_refused_teardown_backup_marks_the_suspending_row_terminal() let row = crate::db::get_sandbox(&conn, &sandbox_id_string) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let artifact = crate::db::LocalArtifactRow { local_artifact_id: staged, sandbox_id: sandbox_id_string.clone(), @@ -1234,7 +1234,7 @@ async fn teardown_suspend_terminal_driver_failures_stop_polling_without_release_ conn.execute("DELETE FROM sandboxes WHERE id = ?1", [&id]) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .drive_teardown_suspend(sandbox_id, generation, SandboxEventTrigger::System) .await @@ -1324,7 +1324,7 @@ async fn teardown_suspend_keeps_polling_until_a_deleting_sandbox_commits_its_del use crate::service::tests::service_tests::prepare_teardown_suspend_request; let conn = crate::db::test_conn(); let (id, generation) = deleting_sandbox_with_closed_teardown_suspend(&conn); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let actions = [ crate::proto::TeardownSuspendAction::BeginOrJoin, crate::proto::TeardownSuspendAction::ObserveOnly, @@ -1381,7 +1381,7 @@ async fn teardown_suspend_keeps_polling_until_a_deleting_sandbox_commits_its_del async fn teardown_suspend_terminal_decision_releases_a_delete_committed_after_the_first_snapshot() { let conn = crate::db::test_conn(); let (id, generation) = deleting_sandbox_with_closed_teardown_suspend(&conn); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); { let db = service.db.lock().unwrap(); // The handler's first snapshot: the row is deleting and no receipt @@ -1428,7 +1428,7 @@ async fn teardown_suspend_preserves_retryable_and_superseded_driver_work() { let sandbox_id = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); insert_stranded_teardown_suspend_sandbox(&conn, sandbox_id, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for error in [ Status::unavailable("retry later"), Status::internal("local IO failed"), @@ -1496,7 +1496,7 @@ async fn teardown_suspend_registration_failure_closes_only_the_requested_generat let sandbox_id = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); let failed_artifact = insert_stranded_teardown_suspend_sandbox(&conn, sandbox_id, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let missing = teardown_artifact_backup_status( ArtifactBackupSubmissionError::Begin(pplx_space_common_jobs::JobError::permanent( "registration refused", @@ -1722,7 +1722,7 @@ async fn teardown_suspend_pause_failure_recovery_allows_same_generation_retry() pplx_space_common_core::types::SandboxQuarantineNextAction::RequiresOperatorAttention, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); assert_eq!( service .close_terminal_teardown_suspend(&id, generation) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs index a0cd6517f8..3d60248eb2 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs @@ -45,7 +45,7 @@ fn armed_clone() -> (SandboxManagerService, SandboxId, SandboxCleanupEpisodeId) None, ) .unwrap(); - (SandboxManagerService::new(conn), id, cleanup_id) + (SandboxManagerService::for_test(conn), id, cleanup_id) } #[tokio::test] @@ -185,7 +185,7 @@ async fn pool_clone_restart_preserves_disposal_ownership() { .execute("VACUUM INTO ?1", [database.to_str().unwrap()]) .unwrap(); drop(service); - let service = SandboxManagerService::new(rusqlite::Connection::open(database).unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open(database).unwrap()); let cache = tempfile::tempdir().unwrap(); service .recover_prelaunch_startup_states_in(cache.path(), &Default::default()) @@ -640,7 +640,7 @@ async fn pool_clone_admission_precedes_all_durable_allocation() { (None, tonic::Code::Unavailable, tonic::Code::Unavailable), (Some(true), tonic::Code::Ok, tonic::Code::NotFound), ] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let methods = Arc::new(Mutex::new(Vec::new())); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs b/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs index e3a28f2ae3..0f55eec0b8 100644 --- a/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs @@ -187,7 +187,7 @@ async fn with_agent_gate( conn, SandboxManagerServiceOptions { credential_manager_socket_path: socket, - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); service.network_credentials_enabled = network_credentials_enabled; diff --git a/crates/libs/sbx/mgr/server/src/service/mod.rs b/crates/libs/sbx/mgr/server/src/service/mod.rs index f42685dc99..df1ff0887b 100644 --- a/crates/libs/sbx/mgr/server/src/service/mod.rs +++ b/crates/libs/sbx/mgr/server/src/service/mod.rs @@ -404,6 +404,9 @@ pub struct SandboxManagerService { network_credentials_enabled: bool, node_manager: NodeManagerConnector, template_resolver: Arc, + /// See [`SandboxManagerServiceOptions::cleanup_run_dir`]. + #[cfg(test)] + cleanup_run_dir: Option, gateway_caller_client: pplx_space_node_mgr_client::NodeManagerGrpcClient, gateway_authority: gateway_authority::GatewayAuthority, /// S3 upload config for suspend snapshots. `None` means uploads are disabled. @@ -977,6 +980,10 @@ pub struct SandboxManagerServiceOptions { pub node_manager: NodeManagerConnector, /// Every template-manager `GetTemplate` the service and its subsystems ask. pub template_resolver: Arc, + /// Run directory whose `fc.pid` links stop and delete target selection + /// reads in test builds. `None` reads the fixed sandbox run directory. + #[cfg(test)] + pub(crate) cleanup_run_dir: Option, } impl Default for SandboxManagerServiceOptions { @@ -997,6 +1004,8 @@ impl Default for SandboxManagerServiceOptions { route_ed25519_verification_bundle: None, node_manager: NodeManagerConnector::host(), template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), + #[cfg(test)] + cleanup_run_dir: None, } } } @@ -1578,10 +1587,6 @@ impl SandboxManagerService { Ok(value) } - pub fn new(conn: rusqlite::Connection) -> Self { - Self::new_with_options(conn, SandboxManagerServiceOptions::default()) - } - pub fn new_with_options( conn: rusqlite::Connection, options: SandboxManagerServiceOptions, @@ -1749,6 +1754,8 @@ impl SandboxManagerService { ), node_manager: options.node_manager, template_resolver: options.template_resolver, + #[cfg(test)] + cleanup_run_dir: options.cleanup_run_dir, drain_state, feature_flags, route_ed25519_verification_bundle: Arc::new(ArcSwapOption::from( @@ -1988,6 +1995,8 @@ mod storage; mod tail_support; mod template_snapshot_support; #[cfg(test)] +pub(crate) mod test_options; +#[cfg(test)] pub(crate) mod tests; mod validation; diff --git a/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs b/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs index 35dcf91dbd..1d9cb8c6c5 100644 --- a/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs @@ -9,7 +9,7 @@ async fn dirty_outbox_poison_requests_fatal_service_drain_without_unwinding_call SandboxManagerServiceOptions { shutdown_token: shutdown.clone(), task_tracker: tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let db = Arc::clone(&service.db); @@ -134,7 +134,7 @@ async fn wait_for_state( #[tokio::test(start_paused = true)] async fn auto_lifecycle_scheduler_refreshes_due_work_before_backlog_drains() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let old_ids = (0..12).map(|_| SandboxId::new()).collect::>(); for (index, sandbox_id) in old_ids.iter().enumerate() { insert_due_suspended_sandbox(&service, &sandbox_id.to_string(), 20 - index as u64); @@ -209,7 +209,7 @@ async fn auto_lifecycle_scheduler_preserves_storage_keys_and_drains_admitted_wor format!("urn:uuid:{id}"), ] { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); insert_due_suspended_sandbox(&service, &storage_id, 10); let lock = service.auto_lifecycle.wake_lock(&storage_id); let held = Arc::clone(&lock).lock_owned().await; @@ -248,7 +248,7 @@ async fn auto_lifecycle_error_takes_the_storage_keyed_wake_lock() { let id = SandboxId::new(); let storage_id = id.to_string().to_uppercase(); let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); insert_due_suspended_sandbox(&service, &storage_id, 10); let lock = service.auto_lifecycle.wake_lock(&storage_id); @@ -302,7 +302,7 @@ async fn auto_lifecycle_error_after_row_removal_does_not_scan_inventory() { SandboxState::Suspended, ); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let removed_id = SandboxId::new(); service .db diff --git a/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs b/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs index b164bcfd72..292d093b9b 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs @@ -74,7 +74,7 @@ async fn staging_cleanup_requires_stable_unowned_lifecycle() { LocalArtifactId::new(), ); std::fs::create_dir_all(&unknown).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (id, path) = paused.unwrap(); for owner in 0..3 { let _wake = (owner == 0).then(|| { @@ -130,7 +130,7 @@ async fn staging_cleanup_rediscovery_survives_rollback_failed_deletion_and_resta path.with_file_name(format!("{RETIRED_PREFIX}{artifact_id}")) }) .collect(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); assert!(sweep(&service, cache.path()).is_empty()); assert!(crate::db::rollback_suspending_to_paused(&service.db.lock().unwrap(), &id).unwrap()); // Simulate an unacknowledged deletion by leaving the selected files in @@ -208,7 +208,7 @@ async fn publication_retirement_is_owned_by_the_sweep_in_any_lifecycle_state() { suspend_snapshot_publication::write_full_suspend_artifact(&snapshot_dir, &id, old_id, b"old"); let staging = suspend_snapshot_publication::suspend_snapshot_staging_dir(&sandbox_dir, new_id); suspend_snapshot_publication::write_full_suspend_artifact(&staging, &id, new_id, b"new"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let _wake = service .auto_lifecycle .wake_lock(&id) @@ -319,7 +319,7 @@ async fn sweep_owns_the_hold_of_a_rolled_back_publication() { let empty_canonical = suspend_snapshot_publication::canonical_suspend_snapshot_dir(&interrupted_dir); std::fs::create_dir_all(&empty_canonical).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!( @@ -398,7 +398,7 @@ async fn sweep_keeps_holds_without_durable_supersession() { b, b"b", ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!(swept.is_empty()); @@ -440,7 +440,7 @@ async fn sweep_leaves_holds_alone_while_a_publication_may_be_in_flight() { let garbage = suspend_snapshot_publication::suspend_snapshot_staging_parent_dir(&sandbox_dir) .join(format!("{RETIRED_PREFIX}{}", LocalArtifactId::new())); suspend_snapshot_publication::write_full_suspend_artifact(&garbage, &id, old_id, b"gc"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let wake = service .auto_lifecycle @@ -511,7 +511,7 @@ async fn sweep_releases_holds_once_a_canonical_snapshot_exists() { let in_flight_dir = cache.path().join(&in_flight); let kept = suspend_snapshot_publication::retired_suspend_snapshot_dir(&in_flight_dir, b); suspend_snapshot_publication::write_full_suspend_artifact(&kept, &in_flight, a, b"a"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!( diff --git a/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs b/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs index e81a9d6805..85f49f1e63 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs @@ -1066,7 +1066,7 @@ async fn a_momentarily_busy_staging_tree_does_not_charge_the_episode() { conn, crate::service::SandboxManagerServiceOptions { sandbox_runtime_live: Arc::new(|_| true), - ..Default::default() + ..crate::service::SandboxManagerServiceOptions::for_test() }, ); let staging_lock = service.auto_lifecycle.staging_lock(&sandbox_id.to_string()); @@ -3709,7 +3709,7 @@ async fn pass_with_every_episode_due( /// Every field explicit, so no fixture reads the host's node identity. fn bound_test_service(conn: rusqlite::Connection) -> SandboxManagerService { - crate::service::tests::service_tests::service_with_runtime_liveness(conn, false) + crate::service::tests::service_tests::service_with_runtime_probe(conn, false) } fn bound() -> i64 { diff --git a/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs b/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs index 2d50ae82d8..b38997aa11 100644 --- a/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs @@ -14,7 +14,7 @@ const HOST_CONTROL_PROTO: &str = include_str!(concat!( fn test_service() -> SandboxManagerService { let connection = crate::db::test_conn(); - SandboxManagerService::new(connection) + SandboxManagerService::for_test(connection) } #[tokio::test] @@ -519,7 +519,7 @@ async fn blocking_wake_without_client_deadline_has_bounded_passive_wait() { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let manager = SandboxManagerService::new(conn); + let manager = SandboxManagerService::for_test(conn); let result = tokio::time::timeout( std::time::Duration::from_secs(31), manager.wake_with_start_config(sandbox_id.clone(), None, Default::default(), true, None), diff --git a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs index f9c9078a4f..6ed112eda9 100644 --- a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs @@ -30,7 +30,6 @@ use pplx_space_common_svc::space_api_route_ed25519::{ use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::PathBuf; -use std::sync::Arc; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; use tonic::Code; @@ -54,9 +53,7 @@ fn enter_route_identity_test_process(bundle_path: Option<&OsStr>) -> bool { } fn local_node() -> NodeId { - NodeId::from_uuid(uuid::Uuid::from_u128( - 0x019f_8000_0000_7000_8000_0000_0000_00fe, - )) + crate::service::test_options::test_node_id() } /// A service whose background tasks belong to the test and whose sockets live @@ -76,17 +73,9 @@ impl RouteTestService { let service = SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.path().join("host-control.sock"), - credential_manager_socket_path: sockets.path().join("credmgr.sock"), - sandbox_runtime_live: Arc::new(|_| false), shutdown_token: shutdown.clone(), task_tracker: tasks.clone(), - local_node_id: Some(local_node()), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), - template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), + ..SandboxManagerServiceOptions::for_test_in(sockets.path()) }, ); Self { diff --git a/crates/libs/sbx/mgr/server/src/service/start/tests.rs b/crates/libs/sbx/mgr/server/src/service/start/tests.rs index 936ca8e1a2..2b7ed3ad1e 100644 --- a/crates/libs/sbx/mgr/server/src/service/start/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/start/tests.rs @@ -12,7 +12,7 @@ use pplx_space_common_core::types::ExecutionGeneration; async fn rootfs_rate_limiter_uses_the_assignment_and_configured_burst() { let service = SandboxManagerService::new_with_config( crate::db::test_conn(), - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { disk_io_burst_secs: std::num::NonZeroU32::new(3), ..Default::default() @@ -696,8 +696,9 @@ async fn restore_releases_artifact_lock_while_waiting_for_cleanup_completion() { // Each refusal happens before Start persists anything or contacts // node-manager, so the row keeps the state the caller observed. Rows that carry -// a template hash would otherwise continue to template-manager, which has no -// test double; only the hash-less row is meant to reach template resolution. +// a template hash would otherwise continue to template-manager, which every +// test service fails to reach; only the hash-less row is meant to reach +// template resolution. #[tokio::test(start_paused = true)] async fn start_refusals_before_activation_leave_the_row_state_unchanged() { use crate::service::tests::service_tests::insert_test_sandbox; @@ -762,7 +763,7 @@ async fn start_refusals_before_activation_leave_the_row_state_unchanged() { } let service = SandboxManagerService::new_with_config( conn, - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { hugepages: case.hugepages, ..Default::default() @@ -883,7 +884,7 @@ async fn suspend_restore_dir_keeps_a_previous_scope_it_cannot_cancel() { ) .unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = prepare_suspend_restore_dir( &service, diff --git a/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs b/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs index 26cf34e761..8ab91ad9c1 100644 --- a/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs @@ -127,7 +127,7 @@ async fn snapshot_observer_refuses_tenant_pool_and_nonrunning_sandboxes() { crate::service::tests::service_tests::insert_test_sandbox(&conn, &id, state); crate::db::update_user(&conn, &id, &owner).unwrap(); crate::db::set_pool_template(&conn, &id, pool).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut request = Request::new(crate::proto::WaitTemplateAppsRequest { sandbox_id: id.clone(), }); diff --git a/crates/libs/sbx/mgr/server/src/service/test_options.rs b/crates/libs/sbx/mgr/server/src/service/test_options.rs new file mode 100644 index 0000000000..ae6d22aabd --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/service/test_options.rs @@ -0,0 +1,85 @@ +//! Service options for tests, with every host dependency redirected. +//! +//! [`SandboxManagerServiceOptions::default`] is production wiring. It reads +//! `SPACE_NODE_ID` or the host's node-id file, names the production +//! host-control and credential-manager sockets, probes `fc.pid` under the +//! fixed sandbox run directory, dials node-manager's fixed socket and asks the +//! host's template-manager. Tests start from +//! [`SandboxManagerServiceOptions::for_test`] instead, and override the fields +//! they exercise with struct update syntax. +//! +//! Service construction still reads process environment of its own, such as +//! the feature-flag provider settings. A test that depends on those runs in a +//! child test process; see `crate::service::tests::enter_route_test_process`. + +use std::path::Path; +use std::sync::Arc; + +use pplx_space_common_core::types::NodeId; + +use super::{SandboxManagerService, SandboxManagerServiceOptions}; + +/// The node identity of every test service. +pub(crate) fn test_node_id() -> NodeId { + NodeId::from_uuid(uuid::Uuid::from_u128( + 0x019f_8000_0000_7000_8000_0000_0000_00fe, + )) +} + +impl SandboxManagerServiceOptions { + /// [`Self::for_test_in`] under a fresh directory that nothing creates, so + /// no runtime is ever live and no socket path names a live service. + pub(crate) fn for_test() -> Self { + Self::for_test_in( + &std::env::temp_dir().join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())), + ) + } + + /// Options whose host dependencies all resolve under `scratch`, a + /// directory the test owns: + /// + /// - the node identity is [`test_node_id`]; + /// - both runtime liveness probes read `scratch//fc.pid`: the + /// service's `sandbox_runtime_live` and stop and delete target + /// selection. A test makes a runtime live by linking that path to an + /// existing file; + /// - the credential-manager and node-manager sockets are paths under + /// `scratch` that nothing binds, so every call to either fails to + /// connect; + /// - the host-control socket is only passed through to node-manager's + /// `Launch`, and also sits under `scratch`, so it names no live host + /// service; + /// - template-manager is a socket under `scratch` that nothing binds, so a + /// dialed lookup fails to connect and a shared-channel lookup is + /// `Unavailable`. + pub(crate) fn for_test_in(scratch: &Path) -> Self { + let run_dir = scratch.to_owned(); + Self { + host_control_socket_path: scratch.join("absent-host-control.sock"), + credential_manager_socket_path: scratch.join("absent-credential-manager.sock"), + sandbox_runtime_live: Arc::new(move |id| { + super::cache_cleanup::is_sandbox_running_in(&run_dir, id) + }), + shutdown_token: tokio_util::sync::CancellationToken::new(), + task_tracker: tokio_util::task::TaskTracker::new(), + local_node_id: Some(test_node_id()), + node_role: None, + node_managed_by: None, + route_ed25519_verification_bundle: None, + node_manager: crate::node_manager_transport::fake::unbound( + &scratch.join("absent-node-manager.sock"), + ), + template_resolver: crate::template_resolver::fake::unbound( + &scratch.join("absent-template-manager.sock"), + ), + cleanup_run_dir: Some(scratch.to_owned()), + } + } +} + +impl SandboxManagerService { + /// A service built from [`SandboxManagerServiceOptions::for_test`]. + pub(crate) fn for_test(conn: rusqlite::Connection) -> Self { + Self::new_with_options(conn, SandboxManagerServiceOptions::for_test()) + } +} diff --git a/crates/libs/sbx/mgr/server/src/service/tests.rs b/crates/libs/sbx/mgr/server/src/service/tests.rs index ee00067d99..0dfbc9dfb0 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests.rs @@ -75,7 +75,7 @@ impl SandboxManagerService { } pub(crate) fn closed_drain_for_test() -> Arc { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); service.begin_draining_for_restart(); Arc::clone(&service.drain_state) } @@ -98,7 +98,7 @@ fn startup_cleanup_discovery_reports_database_failure_instead_of_empty_inventory #[tokio::test] async fn restore_scope_cleanup_rejects_paths_not_owned_by_its_identity_before_remote_calls() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let id = SandboxId::new().to_string(); let scope_id = pplx_space_common_core::types::TransferScopeId::new(); let other_id = SandboxId::new().to_string(); @@ -2572,7 +2572,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new(); insert_test_sandbox(&conn, &sandbox_id.to_string(), SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let current_attempt = SandboxActivationAttemptId::new(); let current_identity = SpaceApiRouteIdentity { @@ -2649,7 +2649,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new(); insert_test_sandbox(&conn, &sandbox_id.to_string(), SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let current_identity = SpaceApiRouteIdentity { route_generation: Some(RouteGeneration::new(4).unwrap()), @@ -3758,7 +3758,7 @@ pub(crate) mod service_tests { None, ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let referencing = service .list_build_referencing_sandboxes(tmplmgr_request( @@ -4411,7 +4411,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); let policy = NetworkPolicyV1::deny_all(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_network_policy(internal_request(SetNetworkPolicyRequest { @@ -4464,7 +4464,7 @@ pub(crate) mod service_tests { ) .unwrap(); let policy = NetworkPolicyV1::deny_all(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_network_policy(internal_request(SetNetworkPolicyRequest { @@ -4504,7 +4504,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = NetworkPolicyV1::deny_all(); let make_request = || { @@ -4604,7 +4604,7 @@ pub(crate) mod service_tests { }, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .apply_network_policy_before_wake(&sandbox_id, "resume") @@ -4674,7 +4674,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4729,7 +4729,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4773,7 +4773,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_auto_lifecycle_policy_exact( @@ -4803,7 +4803,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let metric_capture = pplx_space_metrics::test_capture::capture_counters_guard(); let error = service @@ -4847,7 +4847,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(internal_request(SpacedOpenLoopbackBridgeRequest { @@ -4880,7 +4880,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4919,7 +4919,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(node_ctl_request(SpacedCloseLoopbackBridgeRequest { @@ -4956,7 +4956,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(internal_request_with_invalid_space_api_route_token( @@ -4989,7 +4989,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(internal_request(SpacedCloseLoopbackBridgeRequest { @@ -5021,7 +5021,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(node_ctl_request(SpacedCloseLoopbackBridgeRequest { @@ -5059,7 +5059,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Started); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_wait_ready(node_api_trusted_lifecycle_request(SpacedWaitReadyRequest { @@ -5097,7 +5097,7 @@ pub(crate) mod service_tests { [&sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_wait_ready(node_api_trusted_lifecycle_request(SpacedWaitReadyRequest { @@ -5124,7 +5124,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -5220,7 +5220,7 @@ pub(crate) mod service_tests { }, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for policy in [None, Some(AutoLifecyclePolicy::disabled())] { let error = service @@ -5266,7 +5266,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn wake_rejects_a_generation_fence_without_policy() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let error = service .wake(internal_request(WakeRequest { sandbox_id: SandboxId::new().to_string(), @@ -5393,7 +5393,7 @@ pub(crate) mod service_tests { let row = crate::db::get_sandbox(&conn, &sandbox_id_string) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = service.effective_network_policy(&row).await.unwrap(); @@ -6907,7 +6907,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .evict_cluster_deleted_sandbox(tonic::Request::new( @@ -7020,7 +7020,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { node_role: Some(NodeRole::Worker), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); @@ -7141,7 +7141,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, SandboxState::Started, None); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7165,7 +7165,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7191,7 +7191,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7266,7 +7266,7 @@ pub(crate) mod service_tests { Some(SandboxState::Quarantined) ); } - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7274,8 +7274,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let result = tokio::time::timeout( Duration::from_secs(5), @@ -7333,7 +7331,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let pull_in_flight = service .auto_lifecycle .staging_lock(&sandbox_id) @@ -7573,7 +7571,7 @@ pub(crate) mod service_tests { ); set_test_state_changed_at(&conn, &sandbox_id_string, "0"); set_test_row_version(&conn, &sandbox_id_string, "0"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7581,8 +7579,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); tokio::time::timeout( Duration::from_secs(5), @@ -7749,7 +7745,7 @@ pub(crate) mod service_tests { &sandbox_id_string, &crate::sandbox_eviction::now_secs().to_string(), ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7757,8 +7753,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); // A write has landed before the RPC starts, and the toucher keeps // writing through it, so the row version has moved since the setup @@ -7827,7 +7821,7 @@ pub(crate) mod service_tests { // Old by the row version, fresh by the state clock: the inverse of // the stranded row, and it must be read the other way round. set_test_row_version(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let refusal = service .evict_cluster_deleted_sandbox(root_peer_request(&sandbox_id_string)) @@ -7879,7 +7873,7 @@ pub(crate) mod service_tests { ); set_test_state_changed_at(&conn, &sandbox_id_string, "not-a-timestamp"); set_test_row_version(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7922,7 +7916,7 @@ pub(crate) mod service_tests { ) .unwrap(); set_test_state_changed_at(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7962,7 +7956,7 @@ pub(crate) mod service_tests { .unwrap(); } set_test_row_version(&conn, &sandbox_id_string, "0"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8167,7 +8161,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8185,7 +8179,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, state, Some("seed")); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched(&service, &sandbox_id, state, "pool slot").await; } @@ -8208,7 +8202,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, state, None); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.cluster_deleted_eviction_quiescence = super::eviction::cluster_deleted_eviction_quiescence(configured); @@ -8247,7 +8241,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, SandboxState::Suspended, Some("seed")); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let refusal = service .evict_cluster_deleted_sandbox(root_peer_request(&sandbox_id)) @@ -8267,7 +8261,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8288,7 +8282,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert!(!service.feature_flags.is_configured()); // Stub only node-manager dispatch; run the real eviction claim, @@ -8299,8 +8293,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let result = tokio::time::timeout( Duration::from_secs(5), @@ -8370,7 +8362,7 @@ pub(crate) mod service_tests { ), }; crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .update_runtime_fields(sandbox_id, runtime_fields(generation, 7)) @@ -8576,7 +8568,7 @@ pub(crate) mod service_tests { let typed_sandbox_id = SandboxId::new(); let sandbox_id = typed_sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let subscriber = tracing_subscriber::registry().with(capture.clone()); let guard = tracing::subscriber::set_default(subscriber); @@ -8642,21 +8634,15 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { sandbox_runtime_live: Arc::new(move |_| runtime_live), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ) } + /// Teardown tests compare route identities against this node's identity, + /// [`crate::service::test_options::test_node_id`]. pub(crate) fn service_for_teardown_test(conn: rusqlite::Connection) -> SandboxManagerService { - SandboxManagerService::new_with_options( - conn, - SandboxManagerServiceOptions { - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - ..Default::default() - }, - ) + SandboxManagerService::for_test(conn) } fn route_identity( @@ -10237,7 +10223,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let error = service @@ -10278,7 +10264,7 @@ pub(crate) mod service_tests { "injected cgroup cleanup failure", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let error = service @@ -10321,7 +10307,7 @@ pub(crate) mod service_tests { None, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let quarantined = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -10382,7 +10368,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .delete(internal_request(DeleteRequest { @@ -10402,7 +10388,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id_string).await; let cleanup_lock = service.cleanup_lock(&sandbox_id_string); @@ -10471,7 +10457,7 @@ pub(crate) mod service_tests { [&sandbox_id_string], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let admission = service .admit_lifecycle_cleanup( @@ -10503,7 +10489,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for selector in ["all", "", "not-a-uuid", sandbox_id.get(..8).unwrap()] { for delete in [false, true] { let code = if delete { @@ -10785,7 +10771,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { node_role: Some(NodeRole::Worker), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); @@ -10906,7 +10892,7 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let result = service .admit_lifecycle_cleanup_from_observed_state( @@ -10985,7 +10971,8 @@ pub(crate) mod service_tests { #[tokio::test] async fn unpersisted_delete_clears_activity_without_replacing_live_lock_identity() { - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = + SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let sandbox_id = "sbx-delete-persistence-failure"; let wake_before = service.auto_lifecycle.wake_lock(sandbox_id); service.auto_lifecycle.record_wake_completed(sandbox_id); @@ -12299,7 +12286,7 @@ pub(crate) mod service_tests { SandboxManagerServiceOptions { shutdown_token: shutdown_token.clone(), task_tracker: task_tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let mut wake_guards = Vec::new(); @@ -12358,7 +12345,7 @@ pub(crate) mod service_tests { SandboxManagerServiceOptions { shutdown_token: shutdown_token.clone(), task_tracker: task_tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id.to_string()); @@ -12387,15 +12374,8 @@ pub(crate) mod service_tests { drop(wake_guard); } - pub(crate) fn service_with_runtime_liveness( - conn: rusqlite::Connection, - runtime_live: bool, - ) -> SandboxManagerService { - SandboxManagerService::new_with_options(conn, runtime_liveness_options(runtime_live)) - } - - /// [`service_with_runtime_liveness`] with its absent host-control and - /// credential-manager socket paths under the caller's `sockets` directory. + /// [`service_with_runtime_probe`] with its host dependencies under the + /// caller's `sockets` directory. pub(crate) fn service_with_runtime_liveness_and_absent_sockets( conn: rusqlite::Connection, runtime_live: bool, @@ -12404,37 +12384,12 @@ pub(crate) mod service_tests { SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - ..runtime_liveness_options(runtime_live) + sandbox_runtime_live: Arc::new(move |_| runtime_live), + ..SandboxManagerServiceOptions::for_test_in(sockets) }, ) } - /// Every field is explicit: the `Default` options read the node identity - /// from the host. The host-control and credential-manager sockets sit in a - /// directory nothing creates, so a path that reaches either one fails to - /// connect instead of contacting the host's services. - fn runtime_liveness_options(runtime_live: bool) -> SandboxManagerServiceOptions { - let sockets = - std::env::temp_dir().join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())); - SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(move |_| runtime_live), - shutdown_token: tokio_util::sync::CancellationToken::new(), - task_tracker: tokio_util::task::TaskTracker::new(), - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), - template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), - } - } - fn insert_suspending_resuspend(conn: &rusqlite::Connection, id: &str) -> LocalArtifactId { insert_test_sandbox(conn, id, SandboxState::Paused); let predecessor = LocalArtifactId::new(); @@ -12457,7 +12412,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let predecessor = insert_suspending_resuspend(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); service.record_suspend_snapshot_publication_failure_after_begin( &sandbox_id, "snapshot failed", @@ -12498,7 +12453,7 @@ pub(crate) mod service_tests { ) .unwrap(); let local_artifact_id = LocalArtifactId::new(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let begin_failure = super::grpc_lifecycle::artifact_backup_fence_status( "ClusterApi BeginArtifactBackup unavailable".to_string(), ); @@ -12523,7 +12478,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let snapshot_root = tempfile::tempdir().unwrap(); let local_artifact_id = LocalArtifactId::new(); @@ -12590,7 +12545,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let snapshot_root = tempfile::tempdir().unwrap(); let local_artifact_id = LocalArtifactId::new(); @@ -12684,7 +12639,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = super::grpc_lifecycle::artifact_backup_fence_status( "ClusterApi BeginArtifactBackup unavailable".to_string(), ); @@ -12716,7 +12671,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "publish failed"); @@ -12797,7 +12752,7 @@ pub(crate) mod service_tests { .unwrap() ); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = ::state( @@ -12844,7 +12799,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &malformed_core_id, SandboxState::Created); insert_test_sandbox(&conn, &malformed_provenance_id, SandboxState::Created); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service.record_suspend_snapshot_publication_failure_after_begin( &quarantined_id, "publish failed", @@ -13009,7 +12964,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "publish failed"); { @@ -13055,7 +13010,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); crate::db::update_snapshots_enabled(&conn, &sandbox_id, true).unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13093,7 +13048,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let stale_row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13133,7 +13088,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); // Runtime gone → no re-arm even though armed. - let dead = service_with_runtime_liveness(conn, false); + let dead = service_with_runtime_probe(conn, false); let row = { let db = dead.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13151,7 +13106,7 @@ pub(crate) mod service_tests { let live_conn = crate::db::test_conn(); let sandbox_id2 = SandboxId::new().to_string(); insert_test_sandbox(&live_conn, &sandbox_id2, SandboxState::Suspending); - let live = service_with_runtime_liveness(live_conn, true); + let live = service_with_runtime_probe(live_conn, true); let row2 = { let db = live.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id2).unwrap().unwrap() @@ -13183,7 +13138,7 @@ pub(crate) mod service_tests { b"complete-staging", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let disposition = service.record_suspend_snapshot_publication_failure_after_staging( &sandbox_id, &sandbox_dir, @@ -13242,7 +13197,7 @@ pub(crate) mod service_tests { b"complete-canonical", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let disposition = service.record_suspend_snapshot_publication_failure_after_staging( &sandbox_id, &sandbox_dir, @@ -13272,7 +13227,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "stale failure"); @@ -13349,7 +13304,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn restart_drain_rejects_new_writers_and_waits_for_active_transitions() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let writer = service.state_writer_lease("Start").unwrap(); assert_eq!(service.active_state_transition_count(), 1); @@ -13389,7 +13344,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn restart_drain_reports_active_state_transition_methods() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let first_start = service.state_writer_lease("Start").unwrap(); let wake = service.state_writer_lease("Wake").unwrap(); @@ -13420,8 +13375,6 @@ pub(crate) mod service_tests { let service = service_for_teardown_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new(|_, _| Box::pin(async { Ok(()) }))); // Admit the real blocking RPC, but prevent its durable attempt from @@ -13537,7 +13490,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn dependency_restart_gate_blocks_new_writers_before_draining_active_writer() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let active_writer = service.state_writer_lease("Start").unwrap(); @@ -13567,7 +13520,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let cleanup_blocker = service.cleanup_lock(&sandbox_id).lock_owned().await; @@ -13588,8 +13541,6 @@ pub(crate) mod service_tests { drop(cleanup_blocker); let dispatched = Arc::new(tokio::sync::Notify::new()); let release = Arc::new(tokio::sync::Notify::new()); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new({ let dispatched = dispatched.clone(); let release = release.clone(); @@ -13648,7 +13599,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let startup_recovery = service.startup_recovery_lease().unwrap(); @@ -13757,11 +13708,11 @@ pub(crate) mod service_tests { async fn dependency_restart_capability_hands_off_between_active_instances() { let predecessor = { let conn = crate::db::test_conn(); - SandboxManagerService::new(conn) + SandboxManagerService::for_test(conn) }; let successor = { let conn = crate::db::test_conn(); - SandboxManagerService::new(conn) + SandboxManagerService::for_test(conn) }; let locks = DependencyDrainFiles::new(); locks.enable_for(&predecessor); @@ -13792,7 +13743,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn startup_recovery_waits_for_dependency_replacement_to_finish() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let admission_file = locks.lock_admission(); @@ -13831,7 +13782,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn startup_recovery_admission_wait_is_cancellable() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let admission_file = locks.lock_admission(); @@ -13861,7 +13812,7 @@ pub(crate) mod service_tests { #[tokio::test(start_paused = true)] async fn startup_recovery_admission_wait_has_a_bounded_timeout() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let _admission_file = locks.lock_admission(); @@ -13906,7 +13857,7 @@ pub(crate) mod service_tests { recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let _admission_file = locks.lock_admission(); @@ -13954,7 +13905,7 @@ pub(crate) mod service_tests { let peer_pid = u32::try_from(peer_cred.pid().unwrap()).unwrap(); crate::db::update_netgtw_pid(&conn, &sandbox_id, peer_cred.uid(), peer_pid).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13996,7 +13947,7 @@ pub(crate) mod service_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn restart_drain_observes_concurrent_writer_completion() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let writers = (0..64) .map(|_| service.state_writer_lease("Start").unwrap()) .collect::>(); @@ -14046,7 +13997,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Creating); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14071,7 +14022,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14099,7 +14050,7 @@ pub(crate) mod service_tests { std::fs::write(instance_dir.join("rootfs.ext4"), []).unwrap(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14128,7 +14079,7 @@ pub(crate) mod service_tests { ) .unwrap(); insert_test_sandbox(&conn, &sibling, SandboxState::Checkpointing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_interrupted_resident_operations() .await @@ -14191,7 +14142,7 @@ pub(crate) mod service_tests { [&transient], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.recover_interrupted_startup_states().await.unwrap(); assert_eq!( service.lookup_sandbox(&good).unwrap().state, @@ -14240,7 +14191,7 @@ pub(crate) mod service_tests { BEGIN SELECT RAISE(ABORT, 'injected recovery persistence failure'); END;", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .recover_interrupted_resident_operations() .await @@ -14269,7 +14220,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Resuming); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&id).unwrap(); let error = service .prepare_startup_resident(&row, SandboxState::Started) @@ -14340,7 +14291,7 @@ pub(crate) mod service_tests { let _subscriber = tracing::subscriber::set_default( tracing_subscriber::registry().with(capture.clone()), ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = service.lookup_sandbox(&id).unwrap(); let pid = before.pid; let mut observations = observations.into_iter(); @@ -14423,7 +14374,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &id, SandboxState::Resuming); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let _subscriber = tracing::subscriber::set_default( tracing_subscriber::registry().with(capture.clone()), @@ -14476,7 +14427,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &id, SandboxState::Checkpointing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = service.lookup_sandbox(&id).unwrap(); let record = service .recover_resident_with_probe( @@ -14527,7 +14478,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Pausing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&sandbox_id).unwrap(); let probe_count = Arc::new(AtomicUsize::new(0)); @@ -14561,7 +14512,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Pausing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&sandbox_id).unwrap(); let observations = Arc::new(Mutex::new( [ @@ -14611,7 +14562,7 @@ pub(crate) mod service_tests { .unwrap(); let stopping_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &stopping_id, SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service.recover_interrupted_startup_states().await.unwrap(); @@ -14650,7 +14601,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Paused); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let wake = service.auto_lifecycle.wake_lock(&id).lock_owned().await; let artifact = service.auto_lifecycle.artifact_lock(&id).lock_owned().await; let CleanupLockTryResult::Acquired(delete_fence) = service.try_cleanup_fence(&id) else { @@ -14700,7 +14651,7 @@ pub(crate) mod service_tests { let artifact_id = LocalArtifactId::new(); let staging = suspend_snapshot_staging_dir(&cache.path().join(&id), artifact_id); write_full_suspend_artifact(&staging, &id, artifact_id, b"retained-capture"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let wake = service.auto_lifecycle.wake_lock(&id).lock_owned().await; service .recover_deferred_suspend_publications_in( @@ -14815,7 +14766,7 @@ pub(crate) mod service_tests { Some(bundle) }; let inspector = Arc::new(FakeInspector::new(answer.clone())); - let service = service_with_runtime_liveness(conn, false) + let service = service_with_runtime_probe(conn, false) .with_retained_source_inspector(inspector.clone()); let verdicts = service @@ -14870,7 +14821,7 @@ pub(crate) mod service_tests { ids.push(id); } let service = - service_with_runtime_liveness(conn, false).with_retained_source_inspector(Arc::new( + service_with_runtime_probe(conn, false).with_retained_source_inspector(Arc::new( FakeInspector::new(RetainedSourceCheck::Present { rootfs_bytes: 64 }), )); let mut scan = crate::db::PeriodicLifecycleRecoveryScan::default(); @@ -14910,7 +14861,7 @@ pub(crate) mod service_tests { let artifact = LocalArtifactId::new(); let staging = suspend_snapshot_staging_dir(&cache.path().join(&good), artifact); write_full_suspend_artifact(&staging, &good, artifact, b"retained-capture"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let mut scan = crate::db::PeriodicLifecycleRecoveryScan::default(); for _ in 0..2 { let rows = crate::db::periodic_lifecycle_recovery_batch( @@ -14974,7 +14925,7 @@ pub(crate) mod service_tests { std::os::unix::fs::symlink(target, &link).unwrap(); assert_eq!(std::fs::exists(&link).is_err(), unreadable); } - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let shutdown = CancellationToken::new(); shutdown.cancel(); service @@ -15040,7 +14991,7 @@ pub(crate) mod service_tests { b"staged-artifact", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15098,7 +15049,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let stale_row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -15155,7 +15106,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15188,7 +15139,7 @@ pub(crate) mod service_tests { b"published-before-db-pin", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15230,7 +15181,7 @@ pub(crate) mod service_tests { b"canonical-artifact", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15263,7 +15214,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let receipt_id = SandboxCleanupEpisodeId::new(); let execution_generation = ExecutionGeneration::new(7).unwrap(); @@ -15313,7 +15264,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let local_quarantine_id = Uuid::now_v7(); @@ -15356,7 +15307,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let info = service @@ -15387,7 +15338,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let local_quarantine_id = Uuid::now_v7(); @@ -15452,7 +15403,7 @@ pub(crate) mod service_tests { std::fs::create_dir_all(&staging_dir).unwrap(); std::fs::write(staging_dir.join("vmstate"), b"partial").unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15516,7 +15467,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); // Park the lock so the wake and the suspend queue behind it in a // deterministic order: wake first, suspend second. @@ -15576,7 +15527,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .wake(internal_request(WakeRequest { @@ -15604,7 +15555,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.auto_lifecycle.record_wake_completed(&sandbox_id); tokio::time::sleep(std::time::Duration::from_millis(20)).await; @@ -15643,7 +15594,7 @@ pub(crate) mod service_tests { ) .unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .suspend(internal_request(SuspendRequest { @@ -15745,7 +15696,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = access_idle_last_touched(&service, &sandbox_id); service.begin_draining_for_restart(); @@ -15771,7 +15722,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.begin_draining_for_restart(); let error = match service @@ -15879,7 +15830,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { route_ed25519_verification_bundle: Some(Arc::new(verification_bundle)), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); (service, route_token) @@ -16127,7 +16078,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .set_auto_lifecycle_policy_exact( &id, @@ -16192,7 +16143,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, 1_234); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = access_idle_last_touched(&service, &sandbox_id); let before_access = last_access_at(&service, &sandbox_id); @@ -16234,7 +16185,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .wake(internal_request(WakeRequest { @@ -16308,7 +16259,7 @@ pub(crate) mod service_tests { // Arrange. let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let subscriber = tracing_subscriber::registry().with(capture.clone()); let guard = tracing::subscriber::set_default(subscriber); @@ -16470,7 +16421,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -16501,7 +16452,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for _ in 0..5 { let second = now_secs(); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -16551,7 +16502,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (_host_tx, _client_stream) = open_presence_test_stream(None); yield_to_forward_task().await; @@ -16573,7 +16524,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let seeded_epoch = access_idle_epoch(&service, &sandbox_id); let (host_tx, _client_stream) = @@ -16611,7 +16562,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let _policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let seeded_epoch = access_idle_epoch(&service, &sandbox_id); let (tick_observer, mut ticks) = tokio::sync::mpsc::unbounded_channel(); @@ -16654,7 +16605,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (_host_tx, client_stream) = open_presence_test_stream(Some(test_stream_presence(&service, &sandbox_id, 60, 3600))); @@ -16683,7 +16634,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); // cap = 2 intervals: ticks at +60s and +120s land (the +120s tick // observes elapsed >= cap and disarms), later ticks must not fire. @@ -16720,7 +16671,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); { let db = service.db.lock().expect("mutex poisoned"); crate::db::update_state(&db, &sandbox_id, SandboxState::Suspended) @@ -16752,7 +16703,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let stale = seed_due_access_idle_sandbox_state(&service, &sandbox_id, &policy, 300); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id); @@ -16790,7 +16741,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let stale = seed_due_access_idle_sandbox_state(&service, &sandbox_id, &policy, 300); let (_host_tx, _client_stream) = @@ -16814,7 +16765,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (host_tx, mut client_stream) = open_presence_test_stream(Some(test_stream_presence(&service, &sandbox_id, 60, 3600))); @@ -16875,7 +16826,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for round in 0..32 { // Clears the wake-completion record (and the lock maps) so each @@ -16951,7 +16902,7 @@ pub(crate) mod service_tests { rusqlite::params![stale_updated_at, sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .run_due_auto_lifecycle(&sandbox_id) @@ -17038,7 +16989,7 @@ pub(crate) mod service_tests { rusqlite::params![stale_updated_at, sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .run_due_auto_lifecycle(&sandbox_id) @@ -17071,7 +17022,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id); let wake_guard = wake_lock.lock_owned().await; @@ -17110,7 +17061,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let result = tokio::time::timeout( std::time::Duration::from_secs(60), @@ -17176,7 +17127,7 @@ pub(crate) mod service_tests { // and the still-refused failed variants are pinned by the // `checkpoint_start_keeps_refusing_failed_row_*` tests. let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for state in [SandboxState::Created, SandboxState::Stopped] { let sandbox_id = SandboxId::new().to_string(); @@ -17215,7 +17166,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let before = service.lookup_sandbox(&sandbox_id).unwrap(); let mut request = checkpoint_start_request(sandbox_id.clone(), "suspend", None); @@ -17244,7 +17195,7 @@ pub(crate) mod service_tests { let source_sandbox_id = SandboxId::new().to_string(); let artifact_snapshot_id = SnapshotId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let response = service .start_sandbox( @@ -17302,7 +17253,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let mut service = service_with_runtime_liveness(conn, true); + let mut service = service_with_runtime_probe(conn, true); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 0.001, 1.0), )); @@ -17333,7 +17284,7 @@ pub(crate) mod service_tests { async fn missing_checkpoint_restore_stages_before_activation_admission() { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 100.0, 100.0), )); @@ -17363,7 +17314,10 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = service_with_runtime_liveness(conn, true); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = service_with_runtime_probe(conn, true); service .start_sandbox( @@ -17418,7 +17372,7 @@ pub(crate) mod service_tests { ) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let uuid8 = service.lookup_sandbox(&sandbox_id).unwrap().uuid8; let cleanup_guard = service .activation_cleanup_guard( @@ -17503,7 +17457,7 @@ pub(crate) mod service_tests { insert_test_sandbox_with_runtime(&conn, &sandbox_id_string, SandboxState::Pausing); let stopping_id = SandboxId::new(); insert_test_sandbox(&conn, &stopping_id.to_string(), SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets(); @@ -17582,7 +17536,7 @@ pub(crate) mod service_tests { &cleanup_failed_id.to_string(), SandboxState::CleanupFailed, ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut targets = service.recover_requested_cleanup_targets(); targets.sort_by_key(|target| target.sandbox_id); @@ -17625,7 +17579,7 @@ pub(crate) mod service_tests { rusqlite::params![Uuid::now_v7().to_string(), &sandbox_id_string], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17642,7 +17596,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Deleting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17680,7 +17634,7 @@ pub(crate) mod service_tests { ), }; crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17723,7 +17677,7 @@ pub(crate) mod service_tests { CleanupCompletion::MarkStopped, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let owner = CleanupLockGuard::new( Arc::clone(&service.cleanup_locks), sandbox_id_string.clone(), @@ -17762,7 +17716,7 @@ pub(crate) mod service_tests { CleanupCompletion::Quarantine, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let incumbent = match service.try_cleanup_fence(&sandbox_id_string) { CleanupLockTryResult::Acquired(fence) => fence, CleanupLockTryResult::Busy => panic!("test fence must be available"), @@ -17895,7 +17849,7 @@ pub(crate) mod service_tests { ) .unwrap() .unwrap(); - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 1_000.0, 1.0), )); @@ -17985,7 +17939,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); mark_test_remote_restore_artifact_staging(&conn, &sandbox_id); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 0.001, 1.0), )); @@ -18051,7 +18005,7 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18084,7 +18038,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Failed); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); for _ in 0..2 { let error = service .start_sandbox( @@ -18117,7 +18071,7 @@ pub(crate) mod service_tests { "user requested delete", ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18150,7 +18104,7 @@ pub(crate) mod service_tests { [&sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18184,7 +18138,10 @@ pub(crate) mod service_tests { rusqlite::params![LocalArtifactId::new().to_string(), &sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18224,7 +18181,7 @@ pub(crate) mod service_tests { crate::db::CleanupCompletion::DeleteRow, ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18247,11 +18204,12 @@ pub(crate) mod service_tests { ); } - // Paused, like the sibling checkpoint-retry cases: the `Unavailable` this - // asserts is produced by the vol-manager connect exhausting its retry - // budget, which against an absent socket is ~9.5 s of `tokio::time` - // backoff. The connects still run and still fail; only the backoff between - // them is virtual. + // Paused, like the sibling checkpoint-retry cases. The `Unavailable` this + // asserts comes from restore staging under the fixed cache root: as an + // ordinary user, as CI runs it, the staging directory's `create_dir_all` + // fails; as root, the volume-manager connect that follows exhausts its + // `tokio::time` retry budget against an absent socket, which pausing + // makes virtual. #[tokio::test(start_paused = true)] async fn checkpoint_retry_preserves_row_for_resumable_restore() { // Attempt 1 of a cross-node restore can be cancelled mid-stage (the @@ -18264,7 +18222,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); mark_test_remote_restore_artifact_staging(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18319,7 +18277,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Failed); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let error = service .start_sandbox( @@ -18429,7 +18387,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18456,7 +18414,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let error = service .start_sandbox( @@ -18488,7 +18446,10 @@ pub(crate) mod service_tests { crate::db::CleanupCompletion::MarkSuspended, ) .unwrap(); - let service = SandboxManagerService::new(conn); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = SandboxManagerService::for_test(conn); let _ = service .start_sandbox( @@ -18533,7 +18494,7 @@ pub(crate) mod service_tests { "restore activation was interrupted", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; service.auto_lifecycle.record_wake_completed(&sandbox_id); let pending = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -18603,7 +18564,7 @@ pub(crate) mod service_tests { .unwrap(); let before = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &key, &before).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); install_authoritative_route_epoch( &mut service.route_stream_epochs.lock().unwrap(), id, @@ -18717,7 +18678,7 @@ pub(crate) mod service_tests { .unwrap(); let old = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &id.to_string(), &old).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let protection = { let mut epochs = service.route_stream_epochs.lock().unwrap(); install_authoritative_route_epoch(&mut epochs, id, RouteStreamEpochIdentity(identity)); @@ -18783,7 +18744,7 @@ pub(crate) mod service_tests { .unwrap(); let due = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &key, &due).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); install_authoritative_route_epoch( &mut service.route_stream_epochs.lock().unwrap(), id, @@ -18848,7 +18809,7 @@ pub(crate) mod service_tests { let key = id.to_string(); let policy = access_idle_policy(300); insert_test_sandbox_with_policy(&conn, &key, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut owners = Vec::new(); for generation in [1, 2] { let identity = route_identity(id, generation, SandboxActivationAttemptId::new()); @@ -18910,7 +18871,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::Unfenced, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let now = now_secs(); let mut request = touch_auto_lifecycle_access_request(&id.to_string(), now); let extend = @@ -19059,7 +19020,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle::extend_access_deadline(&mut runtime, &policy, now_secs() + 900) .unwrap(); crate::db::update_auto_lifecycle_state(&conn, &id, &runtime).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .set_auto_lifecycle_policy_exact( &id, @@ -19098,7 +19059,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let policy = access_idle_policy(30); insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; let response = service @@ -19137,7 +19098,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let policy = access_idle_policy(30); insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; service @@ -19186,7 +19147,7 @@ pub(crate) mod service_tests { &AutoLifecycleRuntimeState::default(), ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; let response = service @@ -19222,7 +19183,7 @@ pub(crate) mod service_tests { .unwrap() .expect("sandbox row") .auto_lifecycle_state; - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .touch_auto_lifecycle(internal_request(touch_auto_lifecycle_access_request( @@ -19251,7 +19212,7 @@ pub(crate) mod service_tests { SandboxState::Started, &AutoLifecyclePolicy::disabled(), ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .touch_auto_lifecycle(internal_request(touch_auto_lifecycle_access_request( @@ -19469,7 +19430,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); let outcome = service @@ -19520,7 +19481,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let outcome = service .request_guest_active_for(&sandbox_id, Duration::from_secs(10)) @@ -19552,7 +19513,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); assert!( diff --git a/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs index 4b4ba154bc..858086d167 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs @@ -1,4 +1,4 @@ -use super::service_tests::{insert_test_sandbox, runtime_fields, service_for_teardown_test}; +use super::service_tests::{insert_test_sandbox, runtime_fields}; use super::*; use crate::service::activation_support::internal_request; use pplx_space_common_core::types::ExecutionGeneration; @@ -22,18 +22,20 @@ async fn cancelled_cleanup_rpcs_preserve_admitted_termination_strength() { let sandbox_id = SandboxId::new(); let id = sandbox_id.to_string(); insert_test_sandbox(&conn, &id, SandboxState::Started); - let service = service_for_teardown_test(conn); + let run_dir = tempfile::tempdir().unwrap(); + let service = SandboxManagerService::new_with_options( + conn, + SandboxManagerServiceOptions::for_test_in(run_dir.path()), + ); service .update_runtime_fields( sandbox_id, runtime_fields(ExecutionGeneration::new(1).unwrap(), 7), ) .unwrap(); - let run_dir = tempfile::tempdir().unwrap(); let sandbox_dir = run_dir.path().join(&id); std::fs::create_dir(&sandbox_dir).unwrap(); std::os::unix::fs::symlink(run_dir.path(), sandbox_dir.join("fc.pid")).unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let barrier = Arc::new(crate::service::cleanup_executor::TestAdmissionBarrier { entered: tokio::sync::Notify::new(), release: tokio::sync::Semaphore::new(0), diff --git a/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs index 676f757818..8d852885ce 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs @@ -68,7 +68,7 @@ async fn pending_local_backups_carry_a_concrete_artifact_kind_for_both_origins() let stopped = SandboxId::new().to_string(); insert_test_sandbox(&conn, &suspended, SandboxState::Suspended); insert_test_sandbox(&conn, &stopped, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let suspend_artifact = LocalArtifactId::new(); let stop_artifact = LocalArtifactId::new(); pin_artifact( @@ -130,7 +130,7 @@ async fn a_degraded_sandbox_still_reports_its_pending_artifact() { let conn = crate::db::test_conn(); let degraded = SandboxId::new().to_string(); insert_test_sandbox(&conn, °raded, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let degraded_artifact = LocalArtifactId::new(); pin_artifact( &service, @@ -188,7 +188,7 @@ async fn a_degraded_sandbox_still_reports_its_pending_artifact() { #[tokio::test] async fn pending_local_backups_are_not_readable_by_the_general_observer_set() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut request = Request::new(crate::proto::ListPendingLocalBackupsRequest {}); request.extensions_mut().insert(PeerIdentity::new( users::SPACE_NODEAPI.uid, diff --git a/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs index 879bbdfc5d..0ba568d985 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs @@ -1,9 +1,16 @@ //! Lifecycle handlers driven against an in-process node-manager. //! -//! The service is built with every host dependency redirected: the -//! node-manager connector reaches a [`FakeNodeManager`], the host-control and -//! credential-manager sockets are test-owned paths nothing binds, and no -//! runtime is ever live. +//! The service is built from [`SandboxManagerServiceOptions::for_test`], with +//! every host dependency redirected: +//! +//! - the node-manager connector reaches a [`FakeNodeManager`]; +//! - every template lookup goes to a template-manager fake whose socket is +//! unbound, so a lookup fails to connect (`TemplateLookupError::Connect`, +//! which create reports as `Internal` "cannot connect to template-manager"); +//! - the credential-manager socket is a test-owned path nothing binds; +//! - the host-control socket, which sandbox-manager only passes through to +//! node-manager's `Launch`, names no live host service; +//! - no runtime is ever live. use super::service_tests::{insert_test_sandbox, runtime_fields}; use super::*; @@ -14,52 +21,20 @@ use pplx_space_node_mgr_client::proto::{ CleanupSandboxRequest, CleanupSandboxResponse, StopProcessRequest, StopProcessResponse, }; -/// A service whose node-manager clients all come from `node_manager` and whose -/// template lookups all go to `template_resolver`. -pub(crate) fn isolated_service( +/// A test service whose node-manager clients all come from `node_manager`. +pub(crate) fn service_with_node_manager( conn: rusqlite::Connection, node_manager: NodeManagerConnector, - template_resolver: Arc, - sockets: &Path, ) -> SandboxManagerService { SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(|_| false), - shutdown_token: CancellationToken::new(), - task_tracker: TaskTracker::new(), - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, node_manager, - template_resolver, + ..SandboxManagerServiceOptions::for_test() }, ) } -/// [`isolated_service`] whose template lookups are all refused. -pub(crate) fn service_with_node_manager( - conn: rusqlite::Connection, - node_manager: NodeManagerConnector, - sockets: &Path, -) -> SandboxManagerService { - isolated_service( - conn, - node_manager, - Arc::new( - crate::template_resolver::fake::FakeTemplateResolver::refusing(Status::unavailable( - "template-manager is outside this test", - )), - ), - sockets, - ) -} - #[tokio::test] async fn fatal_kill_signal_failure_still_schedules_admitted_cleanup() { let stops = Arc::new(Mutex::new(Vec::new())); @@ -84,8 +59,7 @@ async fn fatal_kill_signal_failure_still_schedules_admitted_cleanup() { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Started); - let sockets = tempfile::tempdir().unwrap(); - let service = service_with_node_manager(conn, node_manager.connector(), sockets.path()); + let service = service_with_node_manager(conn, node_manager.connector()); service .update_runtime_fields( sandbox_id, diff --git a/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs index da09a3ef8c..d6f4855a90 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs @@ -13,7 +13,7 @@ fn test_ownership() -> GcOwnership { coordinator: crate::auto_lifecycle::AutoLifecycleCoordinator::default(), drain: Arc::new(crate::service::drain::DrainState::default()), shutdown: CancellationToken::new(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), } } @@ -113,11 +113,10 @@ fn full_disk() -> crate::disk_usage::DiskUsage { } } -/// Paused, like the sibling reaper tests: a reaped candidate's staging -/// directory is removed through node-manager, and against an absent socket -/// that connect spends its whole ~9.5 s retry budget on `tokio::time` backoff. -/// Nothing here reads elapsed time — eligibility is decided from the `now_secs` -/// this test passes in — so the clock only has to not be real. +/// Paused, like the sibling reaper tests. A reaped candidate's staging +/// directory is handed to node-manager, which here is an absent socket dialed +/// once. Nothing here reads elapsed time — eligibility is decided from the +/// `now_secs` this test passes in — so the clock only has to not be real. #[tokio::test(start_paused = true)] async fn malformed_artifact_times_are_retained_without_blocking_valid_siblings() { let conn = open_test_db(); diff --git a/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs index 56b73d4fdf..56733c2b65 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs @@ -1013,7 +1013,7 @@ async fn admission_rejection_preserves_completed_restore_work_for_retry() { &stage, target, &error, - &crate::node_manager_transport::NodeManagerConnector::host(), + &crate::node_manager_transport::fake::unbound_in_absent_dir(), ) .await; diff --git a/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs index a20f7911d1..40e2ffa812 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs @@ -143,7 +143,7 @@ fn checkpoint_context(state: SandboxState) -> (CheckpointContext, String) { db: Arc::new(std::sync::Mutex::new(conn)), auto_lifecycle: crate::auto_lifecycle::AutoLifecycleCoordinator::default(), health_lifecycle: crate::watchdog::HealthLifecycleEpochs::default(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), }, sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs index c164544270..fea0e9c866 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs @@ -37,7 +37,7 @@ async fn upload_executor_converges_unstaged_metadata_failure() { let executor = SnapshotUploadExecutor { db: std::sync::Arc::new(std::sync::Mutex::new(open_backfill_test_db())), drain_state: crate::service::tests::closed_drain_for_test(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -185,7 +185,7 @@ async fn executor_rejects_non_snapshot_upload_job_kind() { rusqlite::Connection::open_in_memory().unwrap(), )), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -218,7 +218,7 @@ async fn executor_requires_begin_before_volume_staging() { rusqlite::Connection::open_in_memory().unwrap(), )), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -364,7 +364,7 @@ fn ready_backup_keeps_completed_evidence_when_bookkeeping_misses_execution_deadl executor: SnapshotUploadExecutor { db: std::sync::Arc::clone(&db), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new( crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, @@ -456,7 +456,7 @@ async fn maintenance_retries_failed_pruning_on_the_next_tick() { let worker = build_worker( std::sync::Arc::clone(&db), std::sync::Arc::clone(&drain), - crate::node_manager_transport::NodeManagerConnector::host(), + crate::node_manager_transport::fake::unbound_in_absent_dir(), ); let shutdown = tokio_util::sync::CancellationToken::new(); let maintenance = tokio::spawn(run_maintenance( @@ -1137,7 +1137,7 @@ fn ready_backup_returns_before_the_retained_source_is_inspected() { let executor = SnapshotUploadExecutor { db: std::sync::Arc::clone(&db), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(GatedInspector { gate: std::sync::Arc::clone(&gate), }), diff --git a/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs b/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs index 1fed44511c..178ff87a94 100644 --- a/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs +++ b/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs @@ -1,6 +1,7 @@ //! A scripted template resolver for tests. -use std::sync::{Mutex, PoisonError}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; use pplx_space_template_mgr_client::proto::{GetTemplateRequest, GetTemplateResponse}; @@ -106,3 +107,54 @@ impl TemplateResolver for FakeTemplateResolver { }) } } + +/// A resolver whose lookups all dial `socket`, a test-owned path nothing +/// binds. As when template-manager is down, a dialed lookup fails with a +/// connect error and a shared-channel lookup with `Unavailable`. The dial is +/// made once, without the host client's connect-retry budget. +pub(crate) fn unbound(socket: &Path) -> Arc { + Arc::new(Unbound(socket.to_owned())) +} + +struct Unbound(PathBuf); + +fn endpoint() -> tonic::transport::Endpoint { + tonic::transport::Endpoint::from_static("http://[::]:50051") +} + +fn client( + channel: tonic::transport::Channel, +) -> pplx_space_template_mgr_client::TemplateManagerGrpcClient { + pplx_space_template_mgr_client::proto::template_manager_client::TemplateManagerClient::new( + pplx_space_common_grpc::trace_grpc_client_transport(channel), + ) +} + +#[async_trait::async_trait] +impl TemplateResolver for Unbound { + async fn get_template( + &self, + request: tonic::Request, + ) -> Result, TemplateLookupError> { + let channel = pplx_space_common_grpc::connect_uds_channel_with_endpoint( + "template-manager", + &self.0, + endpoint(), + ) + .await + .map_err(pplx_space_template_mgr_client::ConnectError::from)?; + Ok(client(channel).get_template(request).await?) + } + + async fn get_template_on_shared_channel( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + client(pplx_space_common_grpc::connect_uds_channel_lazy( + &self.0, + endpoint(), + )) + .get_template(request) + .await + } +} diff --git a/crates/libs/sbx/mgr/server/src/watchdog.rs b/crates/libs/sbx/mgr/server/src/watchdog.rs index cc0ba174cf..e1a3ff38d5 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog.rs @@ -808,12 +808,14 @@ impl SandboxWatchdog { ) -> Self { let auto_lifecycle = AutoLifecycleCoordinator::default(); let lifecycle_epochs = HealthLifecycleEpochs::default(); + // The loops tick immediately; their node-manager is an absent socket. + let node_manager = crate::node_manager_transport::fake::unbound_in_absent_dir(); let snapshot_scheduler = Arc::new(SnapshotScheduler::new(Arc::new( crate::snapshot_scheduler::CheckpointContext { db: db.clone(), auto_lifecycle: auto_lifecycle.clone(), health_lifecycle: lifecycle_epochs.clone(), - node_manager: NodeManagerConnector::host(), + node_manager: node_manager.clone(), }, ))); Self::new( @@ -826,7 +828,7 @@ impl SandboxWatchdog { shutdown_token, Arc::new(crate::service::drain::DrainState::default()), task_tracker, - NodeManagerConnector::host(), + node_manager, ) } diff --git a/crates/libs/sbx/mgr/server/src/watchdog/tests.rs b/crates/libs/sbx/mgr/server/src/watchdog/tests.rs index 1f0f70b921..f54e51cbb3 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog/tests.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog/tests.rs @@ -1103,7 +1103,7 @@ async fn pause_failed_recovery_fixture( db: db.clone(), auto_lifecycle: auto_lifecycle.clone(), health_lifecycle: lifecycle_epochs.clone(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), }, ))); snapshot_scheduler