From dbc7b0bc731cf7bb9df04079466163ea0bff5f94 Mon Sep 17 00:00:00 2001 From: Abhinav Chordia Date: Mon, 12 Oct 2026 20:19:07 +0000 Subject: [PATCH] sbx-mgr: build every test service from explicit test options SandboxManagerServiceOptions::default() is production wiring: it reads SPACE_NODE_ID or the host's node-id file, names the production host-control and credential-manager sockets, probes fc.pid under the fixed sandbox run directory, and dials the host's node-manager and template-manager. About two hundred test call sites built the service from it, through SandboxManagerService::new, `..Default::default()`, `new_with_config(conn, Default::default(), ..)` and helpers built on them. A cfg(test) builder now owns test wiring. SandboxManagerServiceOptions::for_test_in(dir) resolves every host dependency under a test-owned directory: - a fixed node id (test_options::test_node_id); - both runtime liveness probes read dir//fc.pid: the service's sandbox_runtime_live, and stop and delete target selection, through a cfg(test) cleanup_run_dir option that replaces the cleanup executor's hand-set run_dir hook; - credential-manager, node-manager and template-manager sockets that nothing binds. node-manager and template-manager clients dial once and fail to connect, as the host clients do when the services are down; - a host-control socket path, which sandbox-manager only passes through to node-manager's Launch, that names no live service. for_test() uses a fresh directory that nothing creates, and SandboxManagerService::for_test(conn) builds a service from it. Tests override the fields they exercise with struct update syntax. Every test-side constructor and helper now goes through the builder: service_for_teardown_test, service_with_runtime_probe and grpc_core's isolated_service keep their names and delegate to it; service_with_runtime_liveness and runtime_liveness_options fold into service_with_runtime_probe; node_manager_transport_tests' isolated_service folds into the builder; route_identity_tests no longer names the host node-manager and template-manager. Default is unchanged for production. SandboxManagerService::new, which production never called, is removed, so a test cannot fall back to the host defaults through it. With Default hidden from test builds, its only remaining user is run.rs's production wiring. The eight tests that set the executor's run directory by hand now get it from the builder. Subsystem test contexts (snapshot GC, upload jobs, scheduler, watchdog, restore) used NodeManagerConnector::host(); three snapshot-GC tests dialed its socket with DeleteCacheArtifact. They now use an unbound connector under a directory nothing creates. The unbound node-manager fake now dials like a real client: connect() and connect_once() dial the absent socket once and fail with a ConnectError, and lazy clients fail at first use. That is what the host connector does on a node whose node-manager is down, without its retry wait. Three checkpoint-restore tests depended on it: the retirement's cleanup cannot start, so the row keeps CleanupRequired. They now say so in a comment. Test follow-ups: - The forwarded-create grpc_core test drops a `!sandbox_exists` assert that held by construction, since create writes no row before template lookup. - Cases whose status code comes from two production branches also match the message: the auto-lifecycle delete's two Internal guards and the pool-clone delete's two FailedPrecondition guards. - node_state_sync::SKIP_NODE_PUBLISH_METADATA is pub(crate), and the skip-publish test uses it. - The completion dirty-ref wait sleeps 5 ms between checks instead of spinning on yield_now. - The node_manager_transport_tests module doc lists the template resolver and the host-control socket's pass-through role. - The sbx-mgr AGENTS.md test guidance points at the builder. The release space-sbxmgr-svc binary's machine code (.text) is identical before and after, with default features and with the Kind feature set. Only source line numbers embedded in location data differ. AI assistance: written with Claude Code. --- crates/libs/sbx/mgr/server/AGENTS.md | 11 +- .../server/src/node_manager_transport/fake.rs | 61 ++- .../sbx/mgr/server/src/node_state_sync.rs | 2 +- crates/libs/sbx/mgr/server/src/run/tests.rs | 6 +- .../mgr/server/src/sandbox_eviction/tests.rs | 14 +- .../src/service/activation_support/tests.rs | 6 +- .../tests/interception_recovery.rs | 2 +- .../mgr/server/src/service/adapter/tests.rs | 2 +- .../server/src/service/auth_lookup/tests.rs | 2 +- .../auth_lookup/wait_ready_policy_tests.rs | 2 +- .../server/src/service/auth_lookup_tests.rs | 2 +- .../service/cache_cleanup/periodic/tests.rs | 9 +- .../server/src/service/cache_cleanup/tests.rs | 4 +- .../server/src/service/cleanup_executor.rs | 4 - .../src/service/cleanup_executor/tests.rs | 16 +- .../mgr/server/src/service/cleanup_targets.rs | 5 +- .../src/service/cleanup_targets/tests.rs | 8 +- .../mgr/server/src/service/create/tests.rs | 2 +- .../src/service/create_support/tests.rs | 2 +- .../src/service/gateway_authority/tests.rs | 4 +- .../grpc_core/create_template_lookup_tests.rs | 19 +- .../mgr/server/src/service/grpc_core/tests.rs | 93 ++-- .../grpc_lifecycle/backup_policy_tests.rs | 12 +- .../grpc_lifecycle/teardown_backup_tests.rs | 4 +- .../src/service/grpc_lifecycle/tests.rs | 24 +- .../src/service/grpc_pool_wait_clone/tests.rs | 6 +- .../server/src/service/host_control/tests.rs | 2 +- crates/libs/sbx/mgr/server/src/service/mod.rs | 17 +- .../mgr/server/src/service/reconcile/tests.rs | 10 +- .../service/recovery/staging_cleanup/tests.rs | 14 +- .../mgr/server/src/service/recovery/tests.rs | 4 +- .../src/service/restart_admission/tests.rs | 4 +- .../src/service/route_identity_tests.rs | 15 +- .../sbx/mgr/server/src/service/start/tests.rs | 11 +- .../server/src/service/template_apps/tests.rs | 2 +- .../mgr/server/src/service/test_options.rs | 85 ++++ .../libs/sbx/mgr/server/src/service/tests.rs | 457 ++++++++---------- .../tests/cancelled_cleanup_rpc_tests.rs | 10 +- .../tests/local_backup_inventory_tests.rs | 6 +- .../tests/node_manager_transport_tests.rs | 56 +-- .../sbx/mgr/server/src/snapshot_gc/tests.rs | 11 +- .../mgr/server/src/snapshot_restore/tests.rs | 2 +- .../server/src/snapshot_scheduler/tests.rs | 2 +- .../server/src/snapshot_upload_jobs/tests.rs | 12 +- .../mgr/server/src/template_resolver/fake.rs | 54 ++- crates/libs/sbx/mgr/server/src/watchdog.rs | 6 +- .../libs/sbx/mgr/server/src/watchdog/tests.rs | 2 +- 47 files changed, 603 insertions(+), 501 deletions(-) create mode 100644 crates/libs/sbx/mgr/server/src/service/test_options.rs diff --git a/crates/libs/sbx/mgr/server/AGENTS.md b/crates/libs/sbx/mgr/server/AGENTS.md index e418ead309..abb6e1b600 100644 --- a/crates/libs/sbx/mgr/server/AGENTS.md +++ b/crates/libs/sbx/mgr/server/AGENTS.md @@ -24,10 +24,13 @@ Follow "Privileged tests and the CI host" in `docs/rust/space-conventions.md`. (`pplx_space_node_mgr_client::connect()`, `connect_once()`, `lazy()`); give the code under test a test-served node-manager. Cancelling `shutdown_token` first protects only code paths that check it before dialing. -- `SandboxManagerServiceOptions::default()` reads `SPACE_NODE_ID` or the - host's node-id file, uses the production host-control and - credential-manager sockets, and probes `fc.pid` under `SANDBOX_RUN_DIR`. Set - every field where a test reaches them; constructors read env regardless. +- Build test services from `SandboxManagerServiceOptions::for_test()` or + `for_test_in(dir)` and override only the fields a test exercises. + `SandboxManagerServiceOptions::default()` is production wiring: it reads + `SPACE_NODE_ID` or the host's node-id file, uses the production host-control + and credential-manager sockets, probes `fc.pid` under `SANDBOX_RUN_DIR`, and + dials the host's node-manager and template-manager. Constructors read env + regardless. - Never call `db::open()`; use `db::test_conn()`. Module moves change log targets: see "Log targets" in diff --git a/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs b/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs index 0c562b8159..f28e6bca24 100644 --- a/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs +++ b/crates/libs/sbx/mgr/server/src/node_manager_transport/fake.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::marker::PhantomData; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use futures_util::future::BoxFuture; @@ -131,21 +131,68 @@ pub(crate) fn fixed_client(client: NodeManagerGrpcClient) -> NodeManagerConnecto } /// A connector whose clients all dial `socket`, a test-owned path nothing -/// binds, so every node-manager call fails before reaching any server. +/// binds, so every node-manager call fails before reaching any server. As on a +/// node whose node-manager is down, `connect` and `connect_once` fail to dial +/// and a lazy client's calls fail with `Unavailable`. `connect` dials once +/// rather than waiting out the host connector's retry budget. pub(crate) fn unbound(socket: &Path) -> NodeManagerConnector { - fixed_client(lazy_client(socket)) + NodeManagerConnector::new(Arc::new(Unbound(socket.to_owned()))) +} + +/// [`unbound`] on a socket in a fresh directory that nothing creates, for +/// tests that build a subsystem context rather than a service. +pub(crate) fn unbound_in_absent_dir() -> NodeManagerConnector { + unbound( + &std::env::temp_dir() + .join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())) + .join("absent-node-manager.sock"), + ) } fn lazy_client(socket: &Path) -> NodeManagerGrpcClient { - let channel = pplx_space_common_grpc::connect_uds_channel_lazy( - socket, - tonic::transport::Endpoint::from_static("http://[::]:50051"), - ); + let channel = pplx_space_common_grpc::connect_uds_channel_lazy(socket, endpoint()); + client(channel) +} + +fn endpoint() -> tonic::transport::Endpoint { + tonic::transport::Endpoint::from_static("http://[::]:50051") +} + +fn client(channel: tonic::transport::Channel) -> NodeManagerGrpcClient { NodeManagerGrpcClient::new(pplx_space_common_svc::bootstrap::wrap_grpc_client_channel( channel, )) } +struct Unbound(PathBuf); + +impl Unbound { + async fn dial(&self) -> Result { + let channel = pplx_space_common_grpc::connect_uds_channel_with_endpoint( + "node-manager", + &self.0, + endpoint(), + ) + .await?; + Ok(client(channel)) + } +} + +#[async_trait::async_trait] +impl NodeManagerTransport for Unbound { + async fn connect(&self) -> Result { + self.dial().await + } + + async fn connect_once(&self) -> Result { + self.dial().await + } + + fn lazy(&self) -> NodeManagerGrpcClient { + lazy_client(&self.0) + } +} + struct InProcess(NodeManagerGrpcClient); #[async_trait::async_trait] diff --git a/crates/libs/sbx/mgr/server/src/node_state_sync.rs b/crates/libs/sbx/mgr/server/src/node_state_sync.rs index 29cdc680d4..eb7c939634 100644 --- a/crates/libs/sbx/mgr/server/src/node_state_sync.rs +++ b/crates/libs/sbx/mgr/server/src/node_state_sync.rs @@ -15,7 +15,7 @@ use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; use tonic::Request; -const SKIP_NODE_PUBLISH_METADATA: &str = "x-space-skip-cluster-sync"; +pub(crate) const SKIP_NODE_PUBLISH_METADATA: &str = "x-space-skip-cluster-sync"; const DIRTY_MARK_INITIAL_RETRY: std::time::Duration = std::time::Duration::from_secs(1); const DIRTY_MARK_MAX_RETRY: std::time::Duration = std::time::Duration::from_secs(300); const DIRTY_OUTBOX_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(1); diff --git a/crates/libs/sbx/mgr/server/src/run/tests.rs b/crates/libs/sbx/mgr/server/src/run/tests.rs index 7d1e882bf4..0446d94f0f 100644 --- a/crates/libs/sbx/mgr/server/src/run/tests.rs +++ b/crates/libs/sbx/mgr/server/src/run/tests.rs @@ -145,7 +145,7 @@ async fn recovery_wait_does_not_hold_the_dependency_work_gate() { for path in [&capability, &admission, &work] { std::fs::File::create(path).unwrap(); } - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); service .enable_dependency_restart_drain(&capability, &admission, &work) .unwrap(); @@ -774,7 +774,7 @@ async fn grpc_writer_during_pre_handoff_drain_returns_retry_on_successor() { let listener = pplx_space_common_svc::bind_unix_socket(&socket_path, UnixSocketOptions::new(0o600)) .unwrap(); - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let rpc_drain_token = CancellationToken::new(); let shutdown_token = CancellationToken::new(); let server_shutdown = shutdown_token.clone(); @@ -821,7 +821,7 @@ async fn grpc_in_flight_request_interrupted_at_handoff_is_not_marked_replayable( let listener = pplx_space_common_svc::bind_unix_socket(&socket_path, UnixSocketOptions::new(0o600)) .unwrap(); - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let rpc_drain_token = CancellationToken::new(); let shutdown_token = CancellationToken::new(); let server_shutdown = shutdown_token.clone(); diff --git a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs index e78a611cc0..675caa8de2 100644 --- a/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs +++ b/crates/libs/sbx/mgr/server/src/sandbox_eviction/tests.rs @@ -841,7 +841,7 @@ async fn evict_tick_below_engage_returns_before_candidate_scan() { [], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.50, @@ -892,7 +892,7 @@ fn resident_rows_count_every_state() { async fn evict_tick_without_disk_usage_fails_without_scanning() { let conn = open_test_db(); insert_sandbox(&conn, "sbx-cold", "suspended", 1_000); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.0, @@ -924,7 +924,7 @@ async fn evict_tick_does_not_connect_when_all_candidates_are_backed_off() { let conn = open_test_db(); insert_sandbox(&conn, "sbx-backed-off", "suspended", 1_000); insert_pinned_artifact(&conn, "sbx-backed-off", &v7(1_000).to_string(), Some("123")); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cfg = EvictionConfig { enabled: true, engage_fraction: 0.50, @@ -1308,7 +1308,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( use crate::state::SandboxState; use pplx_space_common_core::types::SandboxId; - let service = SandboxManagerService::new(open_test_db()); + let service = SandboxManagerService::for_test(open_test_db()); let unknown = service .reupload_legacy_sandbox_for_eviction(&SandboxId::new().to_string()) .await @@ -1329,7 +1329,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( let conn = open_test_db(); let id = SandboxId::new().to_string(); insert_sandbox(&conn, &id, state.as_str(), 0); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let refused = service .reupload_legacy_sandbox_for_eviction(&id) @@ -1350,7 +1350,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( insert_sandbox(&conn, &id, SandboxState::Restoring.as_str(), 0); let service = SandboxManagerService::new_with_config( conn, - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { hugepages: true, ..Default::default() @@ -1370,7 +1370,7 @@ async fn legacy_reupload_refusals_classify_as_state_skips_or_retryable_failures( /// Empty cluster-api wiring is operator-fixable, so eviction retries the /// reupload instead of skipping the sandbox. async fn assert_missing_wiring_reupload_is_retried() { - let service = SandboxManagerService::new(open_test_db()); + let service = SandboxManagerService::for_test(open_test_db()); let refused = service .reupload_legacy_sandbox_for_eviction( &pplx_space_common_core::types::SandboxId::new().to_string(), diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs b/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs index 37d232b255..4ba5c856c6 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support/tests.rs @@ -590,7 +590,7 @@ async fn restore_finalizer_retains_cleanup_until_state_persistence_succeeds() { SandboxCleanupEpisodeId::new(), ) }; - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let temp = tempfile::tempdir().unwrap(); let snapshot_dir = temp.path().join("snapshot"); std::fs::create_dir(&snapshot_dir).unwrap(); @@ -631,7 +631,7 @@ async fn restore_finalizer_retains_cleanup_until_state_persistence_succeeds() { #[tokio::test] async fn cold_finalizer_retains_cleanup_until_state_persistence_succeeds() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let (cleanup_tx, cleanup_rx) = tokio::sync::oneshot::channel(); let cleanup_guard = ActivationCleanupGuard::with_interruption_reason( SandboxRuntimeLabel::parse("runtime1").unwrap(), @@ -655,7 +655,7 @@ async fn cold_finalizer_retains_cleanup_until_state_persistence_succeeds() { #[tokio::test] async fn cold_finalizer_disarms_cleanup_after_state_persistence_succeeds() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let (cleanup_tx, cleanup_rx) = tokio::sync::oneshot::channel(); let cleanup_guard = ActivationCleanupGuard::with_interruption_reason( SandboxRuntimeLabel::parse("runtime1").unwrap(), diff --git a/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs b/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs index f6c12d393c..5bddc2a23f 100644 --- a/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs +++ b/crates/libs/sbx/mgr/server/src/service/activation_support/tests/interception_recovery.rs @@ -200,7 +200,7 @@ async fn exercise_recovery(restore: bool) { SandboxManagerServiceOptions { credential_manager_socket_path: node_socket, shutdown_token: shutdown.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let _gateway = super::super::super::gateway_identity::tests::GatewayNode::attach(&mut service); diff --git a/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs b/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs index 9f1e758312..8fe84a6f9f 100644 --- a/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/adapter/tests.rs @@ -4,7 +4,7 @@ use crate::service::activation_support::internal_request; fn test_service() -> SandboxManagerService { let connection = crate::db::test_conn(); - SandboxManagerService::new(connection) + SandboxManagerService::for_test(connection) } fn expect_authorization_error(result: Result) -> Status { diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs index 13e272ece4..210e7f8413 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup/tests.rs @@ -21,7 +21,7 @@ async fn gateway_identity_mapping_preserves_exact_pid_and_lifecycle_authority() crate::db::update_netgtw_pid(&conn, &id, owner, 4242).unwrap(); let service = SandboxManagerService::new_with_options( conn, - super::super::SandboxManagerServiceOptions::default(), + super::super::SandboxManagerServiceOptions::for_test(), ); for (pid, uid, allowed) in [ (4242, owner, true), diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs index 4ee88fb043..095d9de720 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup/wait_ready_policy_tests.rs @@ -7,7 +7,7 @@ use pplx_space_common_svc::space_api_route::{ }; fn service() -> SandboxManagerService { - SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()) + SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()) } fn node_api_peer() -> AuthorizedUdsPeer { diff --git a/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs b/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs index a6db53a4d0..ddad9fc8a0 100644 --- a/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/auth_lookup_tests.rs @@ -73,7 +73,7 @@ fn fixture_with_claims( rusqlite::Connection::open_in_memory().unwrap(), SandboxManagerServiceOptions { route_ed25519_verification_bundle: Some(Arc::new(bundle)), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let mut request = tonic::Request::new(()); diff --git a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs index a218e7bb59..1d90aaf1b9 100644 --- a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/periodic/tests.rs @@ -16,7 +16,7 @@ async fn periodic_inventory_progresses_in_pages_without_treating_other_pages_as_ } let stale = SandboxId::new(); insert_test_sandbox(&conn, &stale.to_string(), SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut scan = PeriodicInventoryScan::default(); let shutdown = CancellationToken::new(); let paths = SandboxInventoryPaths::new(cache.path(), runtime.path()); @@ -104,7 +104,7 @@ async fn periodic_inventory_names_only_rowless_cache_directories_for_inspection( std::fs::create_dir(cache.path().join(id.to_string())).unwrap(); } insert_test_sandbox(&conn, &owned.to_string(), SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let batch = service .discover_periodic_inventory_in( &mut PeriodicInventoryScan::default(), @@ -190,7 +190,7 @@ async fn orphan_quarantine_survives_inventory_once_its_runtime_is_gone() { Some(SandboxQuarantineReasonCode::OrphanedSandboxRowMissing.as_str()) ); std::fs::remove_dir_all(&sandbox_run_dir).unwrap(); - let service = crate::service::tests::service_tests::service_with_runtime_liveness(conn, false); + let service = crate::service::tests::service_tests::service_with_runtime_probe(conn, false); let before = database_rows(&service.db_conn().unwrap()); reconcile_one_periodic_page(&service, paths); @@ -267,8 +267,7 @@ async fn startup_and_periodic_inventory_remove_only_rows_whose_state_requires_lo .collect::>(); assert!(!expected_removed.is_empty()); - let service = - crate::service::tests::service_tests::service_with_runtime_liveness(conn, false); + let service = crate::service::tests::service_tests::service_with_runtime_probe(conn, false); let removed_or_published = if periodic { reconcile_one_periodic_page(&service, paths); crate::db::list_node_dirty_refs(&service.db_conn().unwrap(), 1024) diff --git a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs index c585f66b62..3022d3afc9 100644 --- a/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cache_cleanup/tests.rs @@ -398,7 +398,7 @@ async fn row_cleanup_retires_policy_only_after_identity_is_deleted() { .unwrap(); episode.id }); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let queue = &service.network_policy_reconcile_queue; queue.enqueue(id.clone()); let old_attempt = queue.take_pending(&Default::default(), 1).pop().unwrap(); @@ -455,7 +455,7 @@ async fn inventory_retires_registries_atomically_and_retries_busy_registrations( let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cache = tempfile::tempdir().unwrap(); let run = tempfile::tempdir().unwrap(); service.network_policy_reconcile_queue.enqueue(id.clone()); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs index c5d09a03c3..47c8bd90a6 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor.rs @@ -42,8 +42,6 @@ pub(super) struct CleanupExecutor { #[cfg(test)] waiter_read_barrier: Mutex>>, #[cfg(test)] - pub(super) run_dir: Mutex>, - #[cfg(test)] pub(super) cache_dir: Mutex>, wake: tokio::sync::Notify, pub(super) active: dashmap::DashMap, @@ -69,8 +67,6 @@ impl Default for CleanupExecutor { #[cfg(test)] waiter_read_barrier: Mutex::new(None), #[cfg(test)] - run_dir: Mutex::new(None), - #[cfg(test)] cache_dir: Mutex::new(None), wake: tokio::sync::Notify::new(), active: dashmap::DashMap::new(), diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs index ae71d8450e..09b0f834b1 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_executor/tests.rs @@ -50,11 +50,9 @@ async fn blocking_stop_follows_a_retryable_attempt_to_durable_completion() { let sandbox_id = SandboxId::new(); let subject = sandbox_id.to_string(); insert_test_sandbox(&conn, &subject, SandboxState::Stopping); - let sockets = tempfile::tempdir().unwrap(); let service = crate::service::tests::node_manager_transport_tests::service_with_node_manager( conn, node_manager.connector(), - sockets.path(), ); let execution_fence = crate::db::node_execution_fence(&service.db.lock().unwrap(), &subject).unwrap(); @@ -565,14 +563,12 @@ async fn lifecycle_and_job_completion_commit_together_and_recover( ); // Virtual time from here on. A case that registered an activation episode // also registered the sandbox with the snapshot scheduler, whose first tick - // opens a node-manager connection; that connect does not watch shutdown, so - // the drain below has to wait out its whole `tokio::time` retry budget - // (~9.5 s against an absent socket) before the task can observe the - // cancellation. Pausing collapses the backoff without changing what the - // drain observes: the connects still run and still fail, and every - // assertion after this point reads state the drain is required to have - // settled. Only the wait is virtual, so everything above runs on the real - // clock exactly as the executor's own timers need. + // opens a node-manager connection; that connect does not watch shutdown. + // The test service's node-manager is an absent socket dialed once, so the + // connect fails at once, and pausing only keeps any later scheduler timer + // from holding the drain on the real clock. Every assertion after this + // point reads state the drain is required to have settled. Everything + // above runs on the real clock exactly as the executor's own timers need. tokio::time::pause(); service.shutdown_token.cancel(); service.task_tracker.close(); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs index 78a89b0152..8de30f8770 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_targets.rs @@ -206,10 +206,7 @@ impl SandboxManagerService { .map_err(|_| Status::unavailable("cleanup storage is shutting down"))?; #[cfg(test)] let run_dir = self - .cleanup_executor - .run_dir - .lock() - .unwrap() + .cleanup_run_dir .clone() .unwrap_or_else(|| PathBuf::from(RUN_DIR)); let cache_dir = self.sandbox_cache_root(); diff --git a/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs b/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs index 854419a817..42dc627dc0 100644 --- a/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/cleanup_targets/tests.rs @@ -6,7 +6,7 @@ use crate::service::tests::service_tests::insert_test_sandbox; async fn inventory_stream_cancels_while_waiting_for_storage_capacity() { use tokio_stream::StreamExt; for reason in ["deadline", "shutdown", "reader"] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let stream_capacity = service.cleanup_target_streams.available_permits(); let cache = tempfile::tempdir().unwrap(); let permits = service.cleanup_io.available_permits(); @@ -159,7 +159,7 @@ async fn streamed_inventory_includes_rowless_targets_and_preserves_bad_metadata( [&malformed], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let cache = tempfile::tempdir().unwrap(); for id in [&persisted, &rowless] { std::fs::create_dir(cache.path().join(id)).unwrap(); @@ -191,7 +191,7 @@ async fn streamed_inventory_includes_rowless_targets_and_preserves_bad_metadata( #[tokio::test] async fn inventory_stream_owns_capacity_and_reports_failure_instead_of_clean_eof() { use tokio_stream::StreamExt; - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let cache = tempfile::tempdir().unwrap(); let deadline = tokio::time::Instant::now() + Duration::from_secs(120); let capacity = service.cleanup_target_streams.available_permits(); @@ -217,7 +217,7 @@ async fn inventory_stream_owns_capacity_and_reports_failure_instead_of_clean_eof assert_eq!(service.cleanup_target_streams.available_permits(), capacity); for failure in ["deadline", "database"] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let deadline = if failure == "deadline" { tokio::time::Instant::now() } else { diff --git a/crates/libs/sbx/mgr/server/src/service/create/tests.rs b/crates/libs/sbx/mgr/server/src/service/create/tests.rs index 0927a36dbd..0bb09e14a2 100644 --- a/crates/libs/sbx/mgr/server/src/service/create/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/create/tests.rs @@ -57,7 +57,7 @@ async fn create_refusals_before_insert_persist_nothing() { crate::db::test_conn(), SandboxManagerServiceOptions { local_node_id: Some(NodeId::new()), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let id = SandboxId::new().to_string(); diff --git a/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs b/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs index 425243bf02..44fe7b1b3a 100644 --- a/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/create_support/tests.rs @@ -351,7 +351,7 @@ async fn create_intent_preserves_immutables_independently_of_pool_occupancy() { ) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for has_snapshot in [false, true] { let snapshot_dir = has_snapshot.then(|| snapshot.path().to_str().unwrap()); for (field, request, incompatible_with_pool) in &cases { diff --git a/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs b/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs index 0f2a744525..21a67afdd5 100644 --- a/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/gateway_authority/tests.rs @@ -12,7 +12,7 @@ struct Fixture { impl Fixture { fn new() -> Self { - Self::with_options(SandboxManagerServiceOptions::default()) + Self::with_options(SandboxManagerServiceOptions::for_test()) } fn with_options(options: SandboxManagerServiceOptions) -> Self { @@ -202,7 +202,7 @@ async fn gateway_authority_uses_the_configured_node_socket() { let listener = tokio::net::UnixListener::bind(&socket).unwrap(); let fixture = Fixture::with_options(SandboxManagerServiceOptions { credential_manager_socket_path: socket, - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }); let expected = fixture.request.csr.clone(); let issue = tower::service_fn(move |request: Request| { diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs index 9e28442e0e..7a0a96d8e0 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs @@ -118,20 +118,20 @@ async fn create_checks_auto_lifecycle_authority_before_template_lookup() { for case in cases { let templates = Arc::new((case.templates)()); let node_manager = FakeNodeManager::default().serve(); - let sockets = tempfile::tempdir().unwrap(); - let service = crate::service::tests::node_manager_transport_tests::isolated_service( + let service = SandboxManagerService::new_with_options( crate::db::test_conn(), - node_manager.connector(), - Arc::clone(&templates) as Arc, - sockets.path(), + SandboxManagerServiceOptions { + node_manager: node_manager.connector(), + template_resolver: Arc::clone(&templates) as _, + ..SandboxManagerServiceOptions::for_test() + }, ); - let sandbox_id = SandboxId::new().to_string(); let mut request = Request::new(CreateRequest { template: "seed".into(), owner: "root@local".into(), runtime: pplx_space_common_core::types::RUNTIME_FIRECRACKER_ALIAS.into(), boot_mode: BootMode::Cold as i32, - sandbox_id: Some(sandbox_id.clone()), + sandbox_id: Some(SandboxId::new().to_string()), auto_lifecycle: Some(Default::default()), auto_lifecycle_authority: case.authority, ..Default::default() @@ -169,10 +169,5 @@ async fn create_checks_auto_lifecycle_authority_before_template_lookup() { assert!(lookup.metadata.is_empty(), "{}", case.name); } assert!(node_manager.calls().is_empty(), "{}", case.name); - assert!( - !crate::db::sandbox_exists(&service.db_conn().unwrap(), &sandbox_id).unwrap(), - "{}", - case.name - ); } } diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs index c8d7ea8eb1..595061be7c 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_core/tests.rs @@ -30,48 +30,27 @@ fn enter_isolated_test_process() -> bool { ]) } -/// A service built from explicit options. `Default` options read the node -/// identity from a fixed host file and probe runtimes under the fixed run -/// directory; here the node identity is fixed, runtime liveness reads -/// `scratch`, the host-control, credential-manager and node-manager sockets -/// are absent paths under `scratch`, and every template lookup is refused. +/// A service whose host dependencies all resolve under `scratch`; see +/// [`SandboxManagerServiceOptions::for_test_in`]. fn isolated_service( conn: rusqlite::Connection, scratch: &std::path::Path, node_role: Option, ) -> SandboxManagerService { - let run_dir = scratch.to_owned(); SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: scratch.join("absent-host-control.sock"), - credential_manager_socket_path: scratch.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(move |id| { - crate::service::cache_cleanup::is_sandbox_running_in(&run_dir, id) - }), - shutdown_token: tokio_util::sync::CancellationToken::new(), - task_tracker: tokio_util::task::TaskTracker::new(), - local_node_id: Some(pplx_space_common_core::types::NodeId::from_uuid( - uuid::Uuid::from_u128(0x019f_8000_0000_7000_8000_0000_0000_00fe), - )), node_role, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::fake::unbound( - &scratch.join("absent-node-manager.sock"), - ), - template_resolver: Arc::new( - crate::template_resolver::fake::FakeTemplateResolver::refusing( - Status::unavailable("template-manager is outside this test"), - ), - ), + ..SandboxManagerServiceOptions::for_test_in(scratch) }, ) } /// An auto-lifecycle authority no proto value names. Create parses it last -/// before template lookup, which would contact the fixed template-manager -/// socket. +/// before template lookup. A lookup past the barrier stops at the test +/// service's template-manager fake, whose socket is unbound: the lookup fails +/// to connect (`TemplateLookupError::Connect`), which create reports as +/// `Internal` "cannot connect to template-manager". const BARRIER_AUTHORITY: i32 = 98; /// The barrier's refusal; no case expects this message for its own refusal. const BARRIER_REFUSAL: &str = "policy authority proto value 98 is unknown"; @@ -290,7 +269,6 @@ async fn forwarded_create_refuses_owner_and_network_policy_authority_before_temp network_policy_authority: case.network_policy_authority.unwrap_or_default(), ..create_refused_before_template_lookup() }; - let sandbox_id = request.sandbox_id.clone().unwrap(); let error = service .create(peer_request( @@ -301,10 +279,6 @@ async fn forwarded_create_refuses_owner_and_network_policy_authority_before_temp .expect_err(case.name); assert_eq!(error.code(), case.code, "{context}: {error}"); assert!(error.message().contains(case.message), "{context}: {error}"); - assert!( - !crate::db::sandbox_exists(&service.db_conn().unwrap(), &sandbox_id).unwrap(), - "{context}" - ); } } } @@ -345,7 +319,6 @@ impl CleanupFixture { ) -> Self { let run_dir = tempfile::tempdir().unwrap(); let service = isolated_service(conn, run_dir.path(), node_role); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let dispatch_permits = Arc::new(tokio::sync::Semaphore::new(dispatch_permits)); let permits = Arc::clone(&dispatch_permits); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new(move |_, _| { @@ -685,7 +658,7 @@ async fn nonblocking_cleanup_returns_once_admitted_and_honors_skip_node_publish( let mut request = peer_request((), Some(root_peer())); if skip_node_publish { request.metadata_mut().insert( - "x-space-skip-cluster-sync", + crate::node_state_sync::SKIP_NODE_PUBLISH_METADATA, tonic::metadata::MetadataValue::from_static("true"), ); } @@ -752,7 +725,7 @@ async fn nonblocking_cleanup_returns_once_admitted_and_honors_skip_node_publish( // completion marks the sandbox dirty just after the job settles. within_cleanup_wait(&context, async { while !fixture.dirty_ids().contains(&id) { - tokio::task::yield_now().await; + tokio::time::sleep(Duration::from_millis(5)).await; } }) .await; @@ -872,7 +845,9 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy uppercase_id: bool, template_filter: Option<&'static str>, expected_policy: &'a AutoLifecyclePolicy, - refusal: Option, + /// The refusal's code and part of its message; one code comes from + /// more than one branch. + refusal: Option<(tonic::Code, &'static str)>, } for case in [ Case { @@ -880,21 +855,27 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy uppercase_id: false, template_filter: Some("seed"), expected_policy: &policy, - refusal: Some(tonic::Code::Internal), + refusal: Some(( + tonic::Code::Internal, + "auto-lifecycle delete requires one exact sandbox id", + )), }, Case { name: "non-canonical id", uppercase_id: true, template_filter: None, expected_policy: &policy, - refusal: Some(tonic::Code::Internal), + refusal: Some(( + tonic::Code::Internal, + "auto-lifecycle delete requires a canonical sandbox id", + )), }, Case { name: "stale policy", uppercase_id: false, template_filter: None, expected_policy: &stale, - refusal: Some(tonic::Code::Aborted), + refusal: Some((tonic::Code::Aborted, "auto-lifecycle delete lost its")), }, Case { name: "current policy", @@ -950,9 +931,10 @@ async fn auto_lifecycle_delete_requires_one_canonical_id_and_the_expected_policy let result = within_cleanup_wait(case.name, delete).await; match case.refusal { - Some(code) => { + Some((code, message)) => { let error = result.unwrap_err(); - assert_eq!(error.code(), code, "{}", case.name); + assert_eq!(error.code(), code, "{}: {error}", case.name); + assert!(error.message().contains(message), "{}: {error}", case.name); // The reconciler retries a lost policy comparison by this key. assert_eq!( error.metadata().contains_key( @@ -1050,7 +1032,9 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: bool, hold_cleanup_fence: bool, blocking: bool, - refusal: Option, + /// The refusal's code and part of its message; one code comes from + /// more than one branch. + refusal: Option<(tonic::Code, &'static str)>, } for case in [ Case { @@ -1061,7 +1045,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "pool clone publication still owns this sandbox", + )), }, Case { name: "template manager after quarantine", @@ -1071,7 +1058,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "clone disposal hold requires an explicit operator delete", + )), }, Case { name: "auto-lifecycle after quarantine", @@ -1081,7 +1071,10 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: true, hold_cleanup_fence: false, blocking: false, - refusal: Some(tonic::Code::FailedPrecondition), + refusal: Some(( + tonic::Code::FailedPrecondition, + "clone disposal hold requires an explicit operator delete", + )), }, Case { name: "operator while cleanup is busy", @@ -1091,7 +1084,7 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() auto_lifecycle: false, hold_cleanup_fence: true, blocking: false, - refusal: Some(tonic::Code::Unavailable), + refusal: Some((tonic::Code::Unavailable, "clone disposal is busy")), }, Case { name: "operator after quarantine", @@ -1170,10 +1163,12 @@ async fn delete_of_a_pool_clone_disposal_requires_an_operator_after_quarantine() let episode = crate::db::cleanup_episode(&fixture.service.db_conn().unwrap(), &id).unwrap(); match case.refusal { - Some(code) => { + Some((code, message)) => { let episode = episode.unwrap(); assert_eq!(episode.id, cleanup_id, "{}", case.name); - assert_eq!(result.unwrap_err().code(), code, "{}", case.name); + let error = result.unwrap_err(); + assert_eq!(error.code(), code, "{}: {error}", case.name); + assert!(error.message().contains(message), "{}: {error}", case.name); assert_eq!(fixture.state(&id), Some(held_state), "{}", case.name); assert_eq!(episode.requested_at, None, "{}", case.name); assert!(fixture.active_cleanup_job(&id).is_none(), "{}", case.name); diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs index 1406d5f6d0..a22d15935f 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/backup_policy_tests.rs @@ -247,7 +247,7 @@ async fn artifact_backup_admission_refuses_sources_it_cannot_back_up() { ); } } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let requested_id = if case == "blank id" { " " } else { &id }; let error = service @@ -272,7 +272,7 @@ async fn artifact_backup_admission_refuses_sources_it_cannot_back_up() { /// marker in the backup refusal. Eviction's reading of the matching reupload /// refusal is pinned against its classifier in `crate::sandbox_eviction`. async fn assert_refused_as_missing_wiring() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let unknown = SandboxId::new().to_string(); let enqueue = service .enqueue_artifact_backup(artifact_backup_request( @@ -425,7 +425,7 @@ async fn durable_artifact_backup_persists_the_submitted_artifact_identity() { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Suspended); - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.snapshot_upload_config = case.default_bucket.map(|b| Arc::new(upload_config(b))); let sandbox = service.lookup_sandbox(&id).unwrap(); let root = tempfile::tempdir().unwrap(); @@ -663,7 +663,7 @@ async fn service_with_template_policy( egress: Egress, generation: u64, ) -> SandboxManagerService { - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = EffectiveNetworkPolicy::new( stored_policy(egress), generation, @@ -874,7 +874,7 @@ async fn set_network_policy_rejects_malformed_requests_without_mutation() { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for (name, request, expected) in [ ( "unknown sandbox", @@ -1083,7 +1083,7 @@ async fn kill_refuses_signals_it_cannot_deliver_before_cleanup_admission() { if let Some(state) = state { insert_test_sandbox(&conn, &id, state); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.begin_draining_for_restart(); (service, id) }; diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs index ec457aac13..2481a25170 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/teardown_backup_tests.rs @@ -113,7 +113,7 @@ async fn teardown_reuses_ready_backup_but_waits_for_active_local_upload() { ) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let pending = service.ensure_teardown_suspend_backup( &row, super::TeardownSuspendBackup { @@ -178,7 +178,7 @@ async fn failed_teardown_defers_repeated_polls_without_holding_driver_capacity() let sandbox = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); insert_stranded_teardown_suspend_sandbox(&conn, sandbox, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .drive_teardown_suspend(sandbox, generation, SandboxEventTrigger::System) .await diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs index 82ba947137..05109afe47 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_lifecycle/tests.rs @@ -37,7 +37,7 @@ async fn set_network_policy_rejects_local_authority_from_forwarded_service_peer( let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let allow_all = pplx_space_net_common::NetworkPolicyV1::allow_all(); let make_request = |uid: u32, gid: u32| { @@ -108,7 +108,7 @@ async fn set_auto_lifecycle_policy_rejects_local_authority_from_forwarded_servic let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let make_request = |uid: u32, gid: u32| { let mut request = tonic::Request::new(SetAutoLifecyclePolicyRequest { @@ -180,7 +180,7 @@ async fn template_notification_matches_inherited_sandboxes_of_both_authorities() .unwrap(); } } - let service = crate::service::SandboxManagerService::new(conn); + let service = crate::service::SandboxManagerService::for_test(conn); let response = service .template_network_policy_updated(internal_request( crate::proto::TemplateNetworkPolicyUpdatedRequest { @@ -715,7 +715,7 @@ fn current_instance_handler_refuses_incomplete_suspend_without_publishing_a_pin( conn.execute("UPDATE sandboxes SET state = 'restoring' WHERE id = ?1", [&id]) .unwrap(); } - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.config.hugepages = case == "unsupported"; let error = service .enqueue_artifact_backup(crate::service::activation_support::internal_request( @@ -962,7 +962,7 @@ async fn suspended_artifact_adoption_race_uses_existing_artifact_id() { assert!( crate::db::adopt_suspended_local_artifact(&conn, sandbox_id, first, snapshot_dir).unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let resolved = service .adopt_or_get_suspended_local_artifact(sandbox_id, second, snapshot_dir) @@ -1116,7 +1116,7 @@ async fn permanently_refused_teardown_backup_marks_the_suspending_row_terminal() let row = crate::db::get_sandbox(&conn, &sandbox_id_string) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let artifact = crate::db::LocalArtifactRow { local_artifact_id: staged, sandbox_id: sandbox_id_string.clone(), @@ -1234,7 +1234,7 @@ async fn teardown_suspend_terminal_driver_failures_stop_polling_without_release_ conn.execute("DELETE FROM sandboxes WHERE id = ?1", [&id]) .unwrap(); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .drive_teardown_suspend(sandbox_id, generation, SandboxEventTrigger::System) .await @@ -1324,7 +1324,7 @@ async fn teardown_suspend_keeps_polling_until_a_deleting_sandbox_commits_its_del use crate::service::tests::service_tests::prepare_teardown_suspend_request; let conn = crate::db::test_conn(); let (id, generation) = deleting_sandbox_with_closed_teardown_suspend(&conn); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let actions = [ crate::proto::TeardownSuspendAction::BeginOrJoin, crate::proto::TeardownSuspendAction::ObserveOnly, @@ -1381,7 +1381,7 @@ async fn teardown_suspend_keeps_polling_until_a_deleting_sandbox_commits_its_del async fn teardown_suspend_terminal_decision_releases_a_delete_committed_after_the_first_snapshot() { let conn = crate::db::test_conn(); let (id, generation) = deleting_sandbox_with_closed_teardown_suspend(&conn); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); { let db = service.db.lock().unwrap(); // The handler's first snapshot: the row is deleting and no receipt @@ -1428,7 +1428,7 @@ async fn teardown_suspend_preserves_retryable_and_superseded_driver_work() { let sandbox_id = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); insert_stranded_teardown_suspend_sandbox(&conn, sandbox_id, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for error in [ Status::unavailable("retry later"), Status::internal("local IO failed"), @@ -1496,7 +1496,7 @@ async fn teardown_suspend_registration_failure_closes_only_the_requested_generat let sandbox_id = SandboxId::new(); let generation = ExecutionGeneration::new(7).unwrap(); let failed_artifact = insert_stranded_teardown_suspend_sandbox(&conn, sandbox_id, generation); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let missing = teardown_artifact_backup_status( ArtifactBackupSubmissionError::Begin(pplx_space_common_jobs::JobError::permanent( "registration refused", @@ -1722,7 +1722,7 @@ async fn teardown_suspend_pause_failure_recovery_allows_same_generation_retry() pplx_space_common_core::types::SandboxQuarantineNextAction::RequiresOperatorAttention, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); assert_eq!( service .close_terminal_teardown_suspend(&id, generation) diff --git a/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs b/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs index a0cd6517f8..3d60248eb2 100644 --- a/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/grpc_pool_wait_clone/tests.rs @@ -45,7 +45,7 @@ fn armed_clone() -> (SandboxManagerService, SandboxId, SandboxCleanupEpisodeId) None, ) .unwrap(); - (SandboxManagerService::new(conn), id, cleanup_id) + (SandboxManagerService::for_test(conn), id, cleanup_id) } #[tokio::test] @@ -185,7 +185,7 @@ async fn pool_clone_restart_preserves_disposal_ownership() { .execute("VACUUM INTO ?1", [database.to_str().unwrap()]) .unwrap(); drop(service); - let service = SandboxManagerService::new(rusqlite::Connection::open(database).unwrap()); + let service = SandboxManagerService::for_test(rusqlite::Connection::open(database).unwrap()); let cache = tempfile::tempdir().unwrap(); service .recover_prelaunch_startup_states_in(cache.path(), &Default::default()) @@ -640,7 +640,7 @@ async fn pool_clone_admission_precedes_all_durable_allocation() { (None, tonic::Code::Unavailable, tonic::Code::Unavailable), (Some(true), tonic::Code::Ok, tonic::Code::NotFound), ] { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let methods = Arc::new(Mutex::new(Vec::new())); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); diff --git a/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs b/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs index e3a28f2ae3..0f55eec0b8 100644 --- a/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/host_control/tests.rs @@ -187,7 +187,7 @@ async fn with_agent_gate( conn, SandboxManagerServiceOptions { credential_manager_socket_path: socket, - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); service.network_credentials_enabled = network_credentials_enabled; diff --git a/crates/libs/sbx/mgr/server/src/service/mod.rs b/crates/libs/sbx/mgr/server/src/service/mod.rs index f42685dc99..df1ff0887b 100644 --- a/crates/libs/sbx/mgr/server/src/service/mod.rs +++ b/crates/libs/sbx/mgr/server/src/service/mod.rs @@ -404,6 +404,9 @@ pub struct SandboxManagerService { network_credentials_enabled: bool, node_manager: NodeManagerConnector, template_resolver: Arc, + /// See [`SandboxManagerServiceOptions::cleanup_run_dir`]. + #[cfg(test)] + cleanup_run_dir: Option, gateway_caller_client: pplx_space_node_mgr_client::NodeManagerGrpcClient, gateway_authority: gateway_authority::GatewayAuthority, /// S3 upload config for suspend snapshots. `None` means uploads are disabled. @@ -977,6 +980,10 @@ pub struct SandboxManagerServiceOptions { pub node_manager: NodeManagerConnector, /// Every template-manager `GetTemplate` the service and its subsystems ask. pub template_resolver: Arc, + /// Run directory whose `fc.pid` links stop and delete target selection + /// reads in test builds. `None` reads the fixed sandbox run directory. + #[cfg(test)] + pub(crate) cleanup_run_dir: Option, } impl Default for SandboxManagerServiceOptions { @@ -997,6 +1004,8 @@ impl Default for SandboxManagerServiceOptions { route_ed25519_verification_bundle: None, node_manager: NodeManagerConnector::host(), template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), + #[cfg(test)] + cleanup_run_dir: None, } } } @@ -1578,10 +1587,6 @@ impl SandboxManagerService { Ok(value) } - pub fn new(conn: rusqlite::Connection) -> Self { - Self::new_with_options(conn, SandboxManagerServiceOptions::default()) - } - pub fn new_with_options( conn: rusqlite::Connection, options: SandboxManagerServiceOptions, @@ -1749,6 +1754,8 @@ impl SandboxManagerService { ), node_manager: options.node_manager, template_resolver: options.template_resolver, + #[cfg(test)] + cleanup_run_dir: options.cleanup_run_dir, drain_state, feature_flags, route_ed25519_verification_bundle: Arc::new(ArcSwapOption::from( @@ -1988,6 +1995,8 @@ mod storage; mod tail_support; mod template_snapshot_support; #[cfg(test)] +pub(crate) mod test_options; +#[cfg(test)] pub(crate) mod tests; mod validation; diff --git a/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs b/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs index 35dcf91dbd..1d9cb8c6c5 100644 --- a/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/reconcile/tests.rs @@ -9,7 +9,7 @@ async fn dirty_outbox_poison_requests_fatal_service_drain_without_unwinding_call SandboxManagerServiceOptions { shutdown_token: shutdown.clone(), task_tracker: tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let db = Arc::clone(&service.db); @@ -134,7 +134,7 @@ async fn wait_for_state( #[tokio::test(start_paused = true)] async fn auto_lifecycle_scheduler_refreshes_due_work_before_backlog_drains() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let old_ids = (0..12).map(|_| SandboxId::new()).collect::>(); for (index, sandbox_id) in old_ids.iter().enumerate() { insert_due_suspended_sandbox(&service, &sandbox_id.to_string(), 20 - index as u64); @@ -209,7 +209,7 @@ async fn auto_lifecycle_scheduler_preserves_storage_keys_and_drains_admitted_wor format!("urn:uuid:{id}"), ] { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); insert_due_suspended_sandbox(&service, &storage_id, 10); let lock = service.auto_lifecycle.wake_lock(&storage_id); let held = Arc::clone(&lock).lock_owned().await; @@ -248,7 +248,7 @@ async fn auto_lifecycle_error_takes_the_storage_keyed_wake_lock() { let id = SandboxId::new(); let storage_id = id.to_string().to_uppercase(); let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); insert_due_suspended_sandbox(&service, &storage_id, 10); let lock = service.auto_lifecycle.wake_lock(&storage_id); @@ -302,7 +302,7 @@ async fn auto_lifecycle_error_after_row_removal_does_not_scan_inventory() { SandboxState::Suspended, ); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let removed_id = SandboxId::new(); service .db diff --git a/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs b/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs index b164bcfd72..292d093b9b 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery/staging_cleanup/tests.rs @@ -74,7 +74,7 @@ async fn staging_cleanup_requires_stable_unowned_lifecycle() { LocalArtifactId::new(), ); std::fs::create_dir_all(&unknown).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (id, path) = paused.unwrap(); for owner in 0..3 { let _wake = (owner == 0).then(|| { @@ -130,7 +130,7 @@ async fn staging_cleanup_rediscovery_survives_rollback_failed_deletion_and_resta path.with_file_name(format!("{RETIRED_PREFIX}{artifact_id}")) }) .collect(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); assert!(sweep(&service, cache.path()).is_empty()); assert!(crate::db::rollback_suspending_to_paused(&service.db.lock().unwrap(), &id).unwrap()); // Simulate an unacknowledged deletion by leaving the selected files in @@ -208,7 +208,7 @@ async fn publication_retirement_is_owned_by_the_sweep_in_any_lifecycle_state() { suspend_snapshot_publication::write_full_suspend_artifact(&snapshot_dir, &id, old_id, b"old"); let staging = suspend_snapshot_publication::suspend_snapshot_staging_dir(&sandbox_dir, new_id); suspend_snapshot_publication::write_full_suspend_artifact(&staging, &id, new_id, b"new"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let _wake = service .auto_lifecycle .wake_lock(&id) @@ -319,7 +319,7 @@ async fn sweep_owns_the_hold_of_a_rolled_back_publication() { let empty_canonical = suspend_snapshot_publication::canonical_suspend_snapshot_dir(&interrupted_dir); std::fs::create_dir_all(&empty_canonical).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!( @@ -398,7 +398,7 @@ async fn sweep_keeps_holds_without_durable_supersession() { b, b"b", ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!(swept.is_empty()); @@ -440,7 +440,7 @@ async fn sweep_leaves_holds_alone_while_a_publication_may_be_in_flight() { let garbage = suspend_snapshot_publication::suspend_snapshot_staging_parent_dir(&sandbox_dir) .join(format!("{RETIRED_PREFIX}{}", LocalArtifactId::new())); suspend_snapshot_publication::write_full_suspend_artifact(&garbage, &id, old_id, b"gc"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let wake = service .auto_lifecycle @@ -511,7 +511,7 @@ async fn sweep_releases_holds_once_a_canonical_snapshot_exists() { let in_flight_dir = cache.path().join(&in_flight); let kept = suspend_snapshot_publication::retired_suspend_snapshot_dir(&in_flight_dir, b); suspend_snapshot_publication::write_full_suspend_artifact(&kept, &in_flight, a, b"a"); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let swept = sweep(&service, cache.path()); assert!( diff --git a/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs b/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs index e81a9d6805..85f49f1e63 100644 --- a/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/recovery/tests.rs @@ -1066,7 +1066,7 @@ async fn a_momentarily_busy_staging_tree_does_not_charge_the_episode() { conn, crate::service::SandboxManagerServiceOptions { sandbox_runtime_live: Arc::new(|_| true), - ..Default::default() + ..crate::service::SandboxManagerServiceOptions::for_test() }, ); let staging_lock = service.auto_lifecycle.staging_lock(&sandbox_id.to_string()); @@ -3709,7 +3709,7 @@ async fn pass_with_every_episode_due( /// Every field explicit, so no fixture reads the host's node identity. fn bound_test_service(conn: rusqlite::Connection) -> SandboxManagerService { - crate::service::tests::service_tests::service_with_runtime_liveness(conn, false) + crate::service::tests::service_tests::service_with_runtime_probe(conn, false) } fn bound() -> i64 { diff --git a/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs b/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs index 2d50ae82d8..b38997aa11 100644 --- a/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/restart_admission/tests.rs @@ -14,7 +14,7 @@ const HOST_CONTROL_PROTO: &str = include_str!(concat!( fn test_service() -> SandboxManagerService { let connection = crate::db::test_conn(); - SandboxManagerService::new(connection) + SandboxManagerService::for_test(connection) } #[tokio::test] @@ -519,7 +519,7 @@ async fn blocking_wake_without_client_deadline_has_bounded_passive_wait() { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let manager = SandboxManagerService::new(conn); + let manager = SandboxManagerService::for_test(conn); let result = tokio::time::timeout( std::time::Duration::from_secs(31), manager.wake_with_start_config(sandbox_id.clone(), None, Default::default(), true, None), diff --git a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs index f9c9078a4f..6ed112eda9 100644 --- a/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/route_identity_tests.rs @@ -30,7 +30,6 @@ use pplx_space_common_svc::space_api_route_ed25519::{ use std::collections::BTreeMap; use std::ffi::OsStr; use std::path::PathBuf; -use std::sync::Arc; use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; use tonic::Code; @@ -54,9 +53,7 @@ fn enter_route_identity_test_process(bundle_path: Option<&OsStr>) -> bool { } fn local_node() -> NodeId { - NodeId::from_uuid(uuid::Uuid::from_u128( - 0x019f_8000_0000_7000_8000_0000_0000_00fe, - )) + crate::service::test_options::test_node_id() } /// A service whose background tasks belong to the test and whose sockets live @@ -76,17 +73,9 @@ impl RouteTestService { let service = SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.path().join("host-control.sock"), - credential_manager_socket_path: sockets.path().join("credmgr.sock"), - sandbox_runtime_live: Arc::new(|_| false), shutdown_token: shutdown.clone(), task_tracker: tasks.clone(), - local_node_id: Some(local_node()), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), - template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), + ..SandboxManagerServiceOptions::for_test_in(sockets.path()) }, ); Self { diff --git a/crates/libs/sbx/mgr/server/src/service/start/tests.rs b/crates/libs/sbx/mgr/server/src/service/start/tests.rs index 936ca8e1a2..2b7ed3ad1e 100644 --- a/crates/libs/sbx/mgr/server/src/service/start/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/start/tests.rs @@ -12,7 +12,7 @@ use pplx_space_common_core::types::ExecutionGeneration; async fn rootfs_rate_limiter_uses_the_assignment_and_configured_burst() { let service = SandboxManagerService::new_with_config( crate::db::test_conn(), - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { disk_io_burst_secs: std::num::NonZeroU32::new(3), ..Default::default() @@ -696,8 +696,9 @@ async fn restore_releases_artifact_lock_while_waiting_for_cleanup_completion() { // Each refusal happens before Start persists anything or contacts // node-manager, so the row keeps the state the caller observed. Rows that carry -// a template hash would otherwise continue to template-manager, which has no -// test double; only the hash-less row is meant to reach template resolution. +// a template hash would otherwise continue to template-manager, which every +// test service fails to reach; only the hash-less row is meant to reach +// template resolution. #[tokio::test(start_paused = true)] async fn start_refusals_before_activation_leave_the_row_state_unchanged() { use crate::service::tests::service_tests::insert_test_sandbox; @@ -762,7 +763,7 @@ async fn start_refusals_before_activation_leave_the_row_state_unchanged() { } let service = SandboxManagerService::new_with_config( conn, - Default::default(), + crate::service::SandboxManagerServiceOptions::for_test(), crate::config::SandboxConfig { hugepages: case.hugepages, ..Default::default() @@ -883,7 +884,7 @@ async fn suspend_restore_dir_keeps_a_previous_scope_it_cannot_cancel() { ) .unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = prepare_suspend_restore_dir( &service, diff --git a/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs b/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs index 26cf34e761..8ab91ad9c1 100644 --- a/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/template_apps/tests.rs @@ -127,7 +127,7 @@ async fn snapshot_observer_refuses_tenant_pool_and_nonrunning_sandboxes() { crate::service::tests::service_tests::insert_test_sandbox(&conn, &id, state); crate::db::update_user(&conn, &id, &owner).unwrap(); crate::db::set_pool_template(&conn, &id, pool).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut request = Request::new(crate::proto::WaitTemplateAppsRequest { sandbox_id: id.clone(), }); diff --git a/crates/libs/sbx/mgr/server/src/service/test_options.rs b/crates/libs/sbx/mgr/server/src/service/test_options.rs new file mode 100644 index 0000000000..ae6d22aabd --- /dev/null +++ b/crates/libs/sbx/mgr/server/src/service/test_options.rs @@ -0,0 +1,85 @@ +//! Service options for tests, with every host dependency redirected. +//! +//! [`SandboxManagerServiceOptions::default`] is production wiring. It reads +//! `SPACE_NODE_ID` or the host's node-id file, names the production +//! host-control and credential-manager sockets, probes `fc.pid` under the +//! fixed sandbox run directory, dials node-manager's fixed socket and asks the +//! host's template-manager. Tests start from +//! [`SandboxManagerServiceOptions::for_test`] instead, and override the fields +//! they exercise with struct update syntax. +//! +//! Service construction still reads process environment of its own, such as +//! the feature-flag provider settings. A test that depends on those runs in a +//! child test process; see `crate::service::tests::enter_route_test_process`. + +use std::path::Path; +use std::sync::Arc; + +use pplx_space_common_core::types::NodeId; + +use super::{SandboxManagerService, SandboxManagerServiceOptions}; + +/// The node identity of every test service. +pub(crate) fn test_node_id() -> NodeId { + NodeId::from_uuid(uuid::Uuid::from_u128( + 0x019f_8000_0000_7000_8000_0000_0000_00fe, + )) +} + +impl SandboxManagerServiceOptions { + /// [`Self::for_test_in`] under a fresh directory that nothing creates, so + /// no runtime is ever live and no socket path names a live service. + pub(crate) fn for_test() -> Self { + Self::for_test_in( + &std::env::temp_dir().join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())), + ) + } + + /// Options whose host dependencies all resolve under `scratch`, a + /// directory the test owns: + /// + /// - the node identity is [`test_node_id`]; + /// - both runtime liveness probes read `scratch//fc.pid`: the + /// service's `sandbox_runtime_live` and stop and delete target + /// selection. A test makes a runtime live by linking that path to an + /// existing file; + /// - the credential-manager and node-manager sockets are paths under + /// `scratch` that nothing binds, so every call to either fails to + /// connect; + /// - the host-control socket is only passed through to node-manager's + /// `Launch`, and also sits under `scratch`, so it names no live host + /// service; + /// - template-manager is a socket under `scratch` that nothing binds, so a + /// dialed lookup fails to connect and a shared-channel lookup is + /// `Unavailable`. + pub(crate) fn for_test_in(scratch: &Path) -> Self { + let run_dir = scratch.to_owned(); + Self { + host_control_socket_path: scratch.join("absent-host-control.sock"), + credential_manager_socket_path: scratch.join("absent-credential-manager.sock"), + sandbox_runtime_live: Arc::new(move |id| { + super::cache_cleanup::is_sandbox_running_in(&run_dir, id) + }), + shutdown_token: tokio_util::sync::CancellationToken::new(), + task_tracker: tokio_util::task::TaskTracker::new(), + local_node_id: Some(test_node_id()), + node_role: None, + node_managed_by: None, + route_ed25519_verification_bundle: None, + node_manager: crate::node_manager_transport::fake::unbound( + &scratch.join("absent-node-manager.sock"), + ), + template_resolver: crate::template_resolver::fake::unbound( + &scratch.join("absent-template-manager.sock"), + ), + cleanup_run_dir: Some(scratch.to_owned()), + } + } +} + +impl SandboxManagerService { + /// A service built from [`SandboxManagerServiceOptions::for_test`]. + pub(crate) fn for_test(conn: rusqlite::Connection) -> Self { + Self::new_with_options(conn, SandboxManagerServiceOptions::for_test()) + } +} diff --git a/crates/libs/sbx/mgr/server/src/service/tests.rs b/crates/libs/sbx/mgr/server/src/service/tests.rs index ee00067d99..0dfbc9dfb0 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests.rs @@ -75,7 +75,7 @@ impl SandboxManagerService { } pub(crate) fn closed_drain_for_test() -> Arc { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); service.begin_draining_for_restart(); Arc::clone(&service.drain_state) } @@ -98,7 +98,7 @@ fn startup_cleanup_discovery_reports_database_failure_instead_of_empty_inventory #[tokio::test] async fn restore_scope_cleanup_rejects_paths_not_owned_by_its_identity_before_remote_calls() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let id = SandboxId::new().to_string(); let scope_id = pplx_space_common_core::types::TransferScopeId::new(); let other_id = SandboxId::new().to_string(); @@ -2572,7 +2572,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new(); insert_test_sandbox(&conn, &sandbox_id.to_string(), SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let current_attempt = SandboxActivationAttemptId::new(); let current_identity = SpaceApiRouteIdentity { @@ -2649,7 +2649,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new(); insert_test_sandbox(&conn, &sandbox_id.to_string(), SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let current_identity = SpaceApiRouteIdentity { route_generation: Some(RouteGeneration::new(4).unwrap()), @@ -3758,7 +3758,7 @@ pub(crate) mod service_tests { None, ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let referencing = service .list_build_referencing_sandboxes(tmplmgr_request( @@ -4411,7 +4411,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); let policy = NetworkPolicyV1::deny_all(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_network_policy(internal_request(SetNetworkPolicyRequest { @@ -4464,7 +4464,7 @@ pub(crate) mod service_tests { ) .unwrap(); let policy = NetworkPolicyV1::deny_all(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_network_policy(internal_request(SetNetworkPolicyRequest { @@ -4504,7 +4504,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = NetworkPolicyV1::deny_all(); let make_request = || { @@ -4604,7 +4604,7 @@ pub(crate) mod service_tests { }, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .apply_network_policy_before_wake(&sandbox_id, "resume") @@ -4674,7 +4674,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4729,7 +4729,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4773,7 +4773,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .set_auto_lifecycle_policy_exact( @@ -4803,7 +4803,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let metric_capture = pplx_space_metrics::test_capture::capture_counters_guard(); let error = service @@ -4847,7 +4847,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(internal_request(SpacedOpenLoopbackBridgeRequest { @@ -4880,7 +4880,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -4919,7 +4919,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(node_ctl_request(SpacedCloseLoopbackBridgeRequest { @@ -4956,7 +4956,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(internal_request_with_invalid_space_api_route_token( @@ -4989,7 +4989,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(internal_request(SpacedCloseLoopbackBridgeRequest { @@ -5021,7 +5021,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Restoring); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_close_loopback_bridge(node_ctl_request(SpacedCloseLoopbackBridgeRequest { @@ -5059,7 +5059,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Started); complete_runtime_endpoint_identity(&conn, &sandbox_id); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_wait_ready(node_api_trusted_lifecycle_request(SpacedWaitReadyRequest { @@ -5097,7 +5097,7 @@ pub(crate) mod service_tests { [&sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_wait_ready(node_api_trusted_lifecycle_request(SpacedWaitReadyRequest { @@ -5124,7 +5124,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .spaced_open_loopback_bridge(node_ctl_request(SpacedOpenLoopbackBridgeRequest { @@ -5220,7 +5220,7 @@ pub(crate) mod service_tests { }, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for policy in [None, Some(AutoLifecyclePolicy::disabled())] { let error = service @@ -5266,7 +5266,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn wake_rejects_a_generation_fence_without_policy() { - let service = SandboxManagerService::new(crate::db::test_conn()); + let service = SandboxManagerService::for_test(crate::db::test_conn()); let error = service .wake(internal_request(WakeRequest { sandbox_id: SandboxId::new().to_string(), @@ -5393,7 +5393,7 @@ pub(crate) mod service_tests { let row = crate::db::get_sandbox(&conn, &sandbox_id_string) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let policy = service.effective_network_policy(&row).await.unwrap(); @@ -6907,7 +6907,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .evict_cluster_deleted_sandbox(tonic::Request::new( @@ -7020,7 +7020,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { node_role: Some(NodeRole::Worker), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); @@ -7141,7 +7141,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, SandboxState::Started, None); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7165,7 +7165,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7191,7 +7191,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7266,7 +7266,7 @@ pub(crate) mod service_tests { Some(SandboxState::Quarantined) ); } - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7274,8 +7274,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let result = tokio::time::timeout( Duration::from_secs(5), @@ -7333,7 +7331,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let pull_in_flight = service .auto_lifecycle .staging_lock(&sandbox_id) @@ -7573,7 +7571,7 @@ pub(crate) mod service_tests { ); set_test_state_changed_at(&conn, &sandbox_id_string, "0"); set_test_row_version(&conn, &sandbox_id_string, "0"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7581,8 +7579,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); tokio::time::timeout( Duration::from_secs(5), @@ -7749,7 +7745,7 @@ pub(crate) mod service_tests { &sandbox_id_string, &crate::sandbox_eviction::now_secs().to_string(), ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let dispatched = Arc::new(std::sync::Mutex::new(Vec::new())); let observed = Arc::clone(&dispatched); @@ -7757,8 +7753,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); // A write has landed before the RPC starts, and the toucher keeps // writing through it, so the row version has moved since the setup @@ -7827,7 +7821,7 @@ pub(crate) mod service_tests { // Old by the row version, fresh by the state clock: the inverse of // the stranded row, and it must be read the other way round. set_test_row_version(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let refusal = service .evict_cluster_deleted_sandbox(root_peer_request(&sandbox_id_string)) @@ -7879,7 +7873,7 @@ pub(crate) mod service_tests { ); set_test_state_changed_at(&conn, &sandbox_id_string, "not-a-timestamp"); set_test_row_version(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7922,7 +7916,7 @@ pub(crate) mod service_tests { ) .unwrap(); set_test_state_changed_at(&conn, &sandbox_id_string, "1000000000"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -7962,7 +7956,7 @@ pub(crate) mod service_tests { .unwrap(); } set_test_row_version(&conn, &sandbox_id_string, "0"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8167,7 +8161,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, state); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8185,7 +8179,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, state, Some("seed")); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched(&service, &sandbox_id, state, "pool slot").await; } @@ -8208,7 +8202,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, state, None); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.cluster_deleted_eviction_quiescence = super::eviction::cluster_deleted_eviction_quiescence(configured); @@ -8247,7 +8241,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_platform_owned_sandbox(&conn, &sandbox_id, SandboxState::Suspended, Some("seed")); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let refusal = service .evict_cluster_deleted_sandbox(root_peer_request(&sandbox_id)) @@ -8267,7 +8261,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert_evict_refused_and_row_untouched( &service, @@ -8288,7 +8282,7 @@ pub(crate) mod service_tests { rusqlite::params![sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); assert!(!service.feature_flags.is_configured()); // Stub only node-manager dispatch; run the real eviction claim, @@ -8299,8 +8293,6 @@ pub(crate) mod service_tests { observed.lock().unwrap().push((id, kill)); Box::pin(async { Ok(()) }) })); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let result = tokio::time::timeout( Duration::from_secs(5), @@ -8370,7 +8362,7 @@ pub(crate) mod service_tests { ), }; crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .update_runtime_fields(sandbox_id, runtime_fields(generation, 7)) @@ -8576,7 +8568,7 @@ pub(crate) mod service_tests { let typed_sandbox_id = SandboxId::new(); let sandbox_id = typed_sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let subscriber = tracing_subscriber::registry().with(capture.clone()); let guard = tracing::subscriber::set_default(subscriber); @@ -8642,21 +8634,15 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { sandbox_runtime_live: Arc::new(move |_| runtime_live), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ) } + /// Teardown tests compare route identities against this node's identity, + /// [`crate::service::test_options::test_node_id`]. pub(crate) fn service_for_teardown_test(conn: rusqlite::Connection) -> SandboxManagerService { - SandboxManagerService::new_with_options( - conn, - SandboxManagerServiceOptions { - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - ..Default::default() - }, - ) + SandboxManagerService::for_test(conn) } fn route_identity( @@ -10237,7 +10223,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let error = service @@ -10278,7 +10264,7 @@ pub(crate) mod service_tests { "injected cgroup cleanup failure", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let error = service @@ -10321,7 +10307,7 @@ pub(crate) mod service_tests { None, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; let quarantined = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -10382,7 +10368,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .delete(internal_request(DeleteRequest { @@ -10402,7 +10388,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id_string).await; let cleanup_lock = service.cleanup_lock(&sandbox_id_string); @@ -10471,7 +10457,7 @@ pub(crate) mod service_tests { [&sandbox_id_string], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let admission = service .admit_lifecycle_cleanup( @@ -10503,7 +10489,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for selector in ["all", "", "not-a-uuid", sandbox_id.get(..8).unwrap()] { for delete in [false, true] { let code = if delete { @@ -10785,7 +10771,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { node_role: Some(NodeRole::Worker), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); @@ -10906,7 +10892,7 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let result = service .admit_lifecycle_cleanup_from_observed_state( @@ -10985,7 +10971,8 @@ pub(crate) mod service_tests { #[tokio::test] async fn unpersisted_delete_clears_activity_without_replacing_live_lock_identity() { - let service = SandboxManagerService::new(rusqlite::Connection::open_in_memory().unwrap()); + let service = + SandboxManagerService::for_test(rusqlite::Connection::open_in_memory().unwrap()); let sandbox_id = "sbx-delete-persistence-failure"; let wake_before = service.auto_lifecycle.wake_lock(sandbox_id); service.auto_lifecycle.record_wake_completed(sandbox_id); @@ -12299,7 +12286,7 @@ pub(crate) mod service_tests { SandboxManagerServiceOptions { shutdown_token: shutdown_token.clone(), task_tracker: task_tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let mut wake_guards = Vec::new(); @@ -12358,7 +12345,7 @@ pub(crate) mod service_tests { SandboxManagerServiceOptions { shutdown_token: shutdown_token.clone(), task_tracker: task_tracker.clone(), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id.to_string()); @@ -12387,15 +12374,8 @@ pub(crate) mod service_tests { drop(wake_guard); } - pub(crate) fn service_with_runtime_liveness( - conn: rusqlite::Connection, - runtime_live: bool, - ) -> SandboxManagerService { - SandboxManagerService::new_with_options(conn, runtime_liveness_options(runtime_live)) - } - - /// [`service_with_runtime_liveness`] with its absent host-control and - /// credential-manager socket paths under the caller's `sockets` directory. + /// [`service_with_runtime_probe`] with its host dependencies under the + /// caller's `sockets` directory. pub(crate) fn service_with_runtime_liveness_and_absent_sockets( conn: rusqlite::Connection, runtime_live: bool, @@ -12404,37 +12384,12 @@ pub(crate) mod service_tests { SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - ..runtime_liveness_options(runtime_live) + sandbox_runtime_live: Arc::new(move |_| runtime_live), + ..SandboxManagerServiceOptions::for_test_in(sockets) }, ) } - /// Every field is explicit: the `Default` options read the node identity - /// from the host. The host-control and credential-manager sockets sit in a - /// directory nothing creates, so a path that reaches either one fails to - /// connect instead of contacting the host's services. - fn runtime_liveness_options(runtime_live: bool) -> SandboxManagerServiceOptions { - let sockets = - std::env::temp_dir().join(format!("space-sbx-mgr-test-{}", uuid::Uuid::now_v7())); - SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(move |_| runtime_live), - shutdown_token: tokio_util::sync::CancellationToken::new(), - task_tracker: tokio_util::task::TaskTracker::new(), - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), - template_resolver: Arc::new(crate::template_resolver::HostTemplateManager), - } - } - fn insert_suspending_resuspend(conn: &rusqlite::Connection, id: &str) -> LocalArtifactId { insert_test_sandbox(conn, id, SandboxState::Paused); let predecessor = LocalArtifactId::new(); @@ -12457,7 +12412,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let predecessor = insert_suspending_resuspend(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); service.record_suspend_snapshot_publication_failure_after_begin( &sandbox_id, "snapshot failed", @@ -12498,7 +12453,7 @@ pub(crate) mod service_tests { ) .unwrap(); let local_artifact_id = LocalArtifactId::new(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let begin_failure = super::grpc_lifecycle::artifact_backup_fence_status( "ClusterApi BeginArtifactBackup unavailable".to_string(), ); @@ -12523,7 +12478,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let snapshot_root = tempfile::tempdir().unwrap(); let local_artifact_id = LocalArtifactId::new(); @@ -12590,7 +12545,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let snapshot_root = tempfile::tempdir().unwrap(); let local_artifact_id = LocalArtifactId::new(); @@ -12684,7 +12639,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = super::grpc_lifecycle::artifact_backup_fence_status( "ClusterApi BeginArtifactBackup unavailable".to_string(), ); @@ -12716,7 +12671,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "publish failed"); @@ -12797,7 +12752,7 @@ pub(crate) mod service_tests { .unwrap() ); } - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = ::state( @@ -12844,7 +12799,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &malformed_core_id, SandboxState::Created); insert_test_sandbox(&conn, &malformed_provenance_id, SandboxState::Created); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service.record_suspend_snapshot_publication_failure_after_begin( &quarantined_id, "publish failed", @@ -13009,7 +12964,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "publish failed"); { @@ -13055,7 +13010,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); crate::db::update_snapshots_enabled(&conn, &sandbox_id, true).unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13093,7 +13048,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let stale_row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13133,7 +13088,7 @@ pub(crate) mod service_tests { insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); // Runtime gone → no re-arm even though armed. - let dead = service_with_runtime_liveness(conn, false); + let dead = service_with_runtime_probe(conn, false); let row = { let db = dead.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13151,7 +13106,7 @@ pub(crate) mod service_tests { let live_conn = crate::db::test_conn(); let sandbox_id2 = SandboxId::new().to_string(); insert_test_sandbox(&live_conn, &sandbox_id2, SandboxState::Suspending); - let live = service_with_runtime_liveness(live_conn, true); + let live = service_with_runtime_probe(live_conn, true); let row2 = { let db = live.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id2).unwrap().unwrap() @@ -13183,7 +13138,7 @@ pub(crate) mod service_tests { b"complete-staging", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let disposition = service.record_suspend_snapshot_publication_failure_after_staging( &sandbox_id, &sandbox_dir, @@ -13242,7 +13197,7 @@ pub(crate) mod service_tests { b"complete-canonical", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let disposition = service.record_suspend_snapshot_publication_failure_after_staging( &sandbox_id, &sandbox_dir, @@ -13272,7 +13227,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Paused); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); service .record_suspend_snapshot_publication_failure_after_begin(&sandbox_id, "stale failure"); @@ -13349,7 +13304,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn restart_drain_rejects_new_writers_and_waits_for_active_transitions() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let writer = service.state_writer_lease("Start").unwrap(); assert_eq!(service.active_state_transition_count(), 1); @@ -13389,7 +13344,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn restart_drain_reports_active_state_transition_methods() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let first_start = service.state_writer_lease("Start").unwrap(); let wake = service.state_writer_lease("Wake").unwrap(); @@ -13420,8 +13375,6 @@ pub(crate) mod service_tests { let service = service_for_teardown_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new(|_, _| Box::pin(async { Ok(()) }))); // Admit the real blocking RPC, but prevent its durable attempt from @@ -13537,7 +13490,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn dependency_restart_gate_blocks_new_writers_before_draining_active_writer() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let active_writer = service.state_writer_lease("Start").unwrap(); @@ -13567,7 +13520,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let cleanup_blocker = service.cleanup_lock(&sandbox_id).lock_owned().await; @@ -13588,8 +13541,6 @@ pub(crate) mod service_tests { drop(cleanup_blocker); let dispatched = Arc::new(tokio::sync::Notify::new()); let release = Arc::new(tokio::sync::Notify::new()); - let run_dir = tempfile::tempdir().unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); *service.cleanup_executor.dispatch.lock().unwrap() = Some(Arc::new({ let dispatched = dispatched.clone(); let release = release.clone(); @@ -13648,7 +13599,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let startup_recovery = service.startup_recovery_lease().unwrap(); @@ -13757,11 +13708,11 @@ pub(crate) mod service_tests { async fn dependency_restart_capability_hands_off_between_active_instances() { let predecessor = { let conn = crate::db::test_conn(); - SandboxManagerService::new(conn) + SandboxManagerService::for_test(conn) }; let successor = { let conn = crate::db::test_conn(); - SandboxManagerService::new(conn) + SandboxManagerService::for_test(conn) }; let locks = DependencyDrainFiles::new(); locks.enable_for(&predecessor); @@ -13792,7 +13743,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn startup_recovery_waits_for_dependency_replacement_to_finish() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let admission_file = locks.lock_admission(); @@ -13831,7 +13782,7 @@ pub(crate) mod service_tests { #[tokio::test] async fn startup_recovery_admission_wait_is_cancellable() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let admission_file = locks.lock_admission(); @@ -13861,7 +13812,7 @@ pub(crate) mod service_tests { #[tokio::test(start_paused = true)] async fn startup_recovery_admission_wait_has_a_bounded_timeout() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let _admission_file = locks.lock_admission(); @@ -13906,7 +13857,7 @@ pub(crate) mod service_tests { recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let locks = DependencyDrainFiles::new(); locks.enable_for(&service); let _admission_file = locks.lock_admission(); @@ -13954,7 +13905,7 @@ pub(crate) mod service_tests { let peer_pid = u32::try_from(peer_cred.pid().unwrap()).unwrap(); crate::db::update_netgtw_pid(&conn, &sandbox_id, peer_cred.uid(), peer_pid).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -13996,7 +13947,7 @@ pub(crate) mod service_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn restart_drain_observes_concurrent_writer_completion() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let writers = (0..64) .map(|_| service.state_writer_lease("Start").unwrap()) .collect::>(); @@ -14046,7 +13997,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Creating); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14071,7 +14022,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14099,7 +14050,7 @@ pub(crate) mod service_tests { std::fs::write(instance_dir.join("rootfs.ext4"), []).unwrap(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Starting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -14128,7 +14079,7 @@ pub(crate) mod service_tests { ) .unwrap(); insert_test_sandbox(&conn, &sibling, SandboxState::Checkpointing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service .recover_interrupted_resident_operations() .await @@ -14191,7 +14142,7 @@ pub(crate) mod service_tests { [&transient], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.recover_interrupted_startup_states().await.unwrap(); assert_eq!( service.lookup_sandbox(&good).unwrap().state, @@ -14240,7 +14191,7 @@ pub(crate) mod service_tests { BEGIN SELECT RAISE(ABORT, 'injected recovery persistence failure'); END;", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .recover_interrupted_resident_operations() .await @@ -14269,7 +14220,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Resuming); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&id).unwrap(); let error = service .prepare_startup_resident(&row, SandboxState::Started) @@ -14340,7 +14291,7 @@ pub(crate) mod service_tests { let _subscriber = tracing::subscriber::set_default( tracing_subscriber::registry().with(capture.clone()), ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = service.lookup_sandbox(&id).unwrap(); let pid = before.pid; let mut observations = observations.into_iter(); @@ -14423,7 +14374,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &id, SandboxState::Resuming); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let _subscriber = tracing::subscriber::set_default( tracing_subscriber::registry().with(capture.clone()), @@ -14476,7 +14427,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &id, SandboxState::Checkpointing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = service.lookup_sandbox(&id).unwrap(); let record = service .recover_resident_with_probe( @@ -14527,7 +14478,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Pausing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&sandbox_id).unwrap(); let probe_count = Arc::new(AtomicUsize::new(0)); @@ -14561,7 +14512,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox_with_runtime(&conn, &sandbox_id, SandboxState::Pausing); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let row = service.lookup_sandbox(&sandbox_id).unwrap(); let observations = Arc::new(Mutex::new( [ @@ -14611,7 +14562,7 @@ pub(crate) mod service_tests { .unwrap(); let stopping_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &stopping_id, SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let records = service.recover_interrupted_startup_states().await.unwrap(); @@ -14650,7 +14601,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, SandboxState::Paused); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let wake = service.auto_lifecycle.wake_lock(&id).lock_owned().await; let artifact = service.auto_lifecycle.artifact_lock(&id).lock_owned().await; let CleanupLockTryResult::Acquired(delete_fence) = service.try_cleanup_fence(&id) else { @@ -14700,7 +14651,7 @@ pub(crate) mod service_tests { let artifact_id = LocalArtifactId::new(); let staging = suspend_snapshot_staging_dir(&cache.path().join(&id), artifact_id); write_full_suspend_artifact(&staging, &id, artifact_id, b"retained-capture"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let wake = service.auto_lifecycle.wake_lock(&id).lock_owned().await; service .recover_deferred_suspend_publications_in( @@ -14815,7 +14766,7 @@ pub(crate) mod service_tests { Some(bundle) }; let inspector = Arc::new(FakeInspector::new(answer.clone())); - let service = service_with_runtime_liveness(conn, false) + let service = service_with_runtime_probe(conn, false) .with_retained_source_inspector(inspector.clone()); let verdicts = service @@ -14870,7 +14821,7 @@ pub(crate) mod service_tests { ids.push(id); } let service = - service_with_runtime_liveness(conn, false).with_retained_source_inspector(Arc::new( + service_with_runtime_probe(conn, false).with_retained_source_inspector(Arc::new( FakeInspector::new(RetainedSourceCheck::Present { rootfs_bytes: 64 }), )); let mut scan = crate::db::PeriodicLifecycleRecoveryScan::default(); @@ -14910,7 +14861,7 @@ pub(crate) mod service_tests { let artifact = LocalArtifactId::new(); let staging = suspend_snapshot_staging_dir(&cache.path().join(&good), artifact); write_full_suspend_artifact(&staging, &good, artifact, b"retained-capture"); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let mut scan = crate::db::PeriodicLifecycleRecoveryScan::default(); for _ in 0..2 { let rows = crate::db::periodic_lifecycle_recovery_batch( @@ -14974,7 +14925,7 @@ pub(crate) mod service_tests { std::os::unix::fs::symlink(target, &link).unwrap(); assert_eq!(std::fs::exists(&link).is_err(), unreadable); } - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let shutdown = CancellationToken::new(); shutdown.cancel(); service @@ -15040,7 +14991,7 @@ pub(crate) mod service_tests { b"staged-artifact", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15098,7 +15049,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspending); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let stale_row = { let db = service.db.lock().expect("mutex poisoned"); crate::db::get_sandbox(&db, &sandbox_id).unwrap().unwrap() @@ -15155,7 +15106,7 @@ pub(crate) mod service_tests { ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15188,7 +15139,7 @@ pub(crate) mod service_tests { b"published-before-db-pin", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15230,7 +15181,7 @@ pub(crate) mod service_tests { b"canonical-artifact", ); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15263,7 +15214,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let receipt_id = SandboxCleanupEpisodeId::new(); let execution_generation = ExecutionGeneration::new(7).unwrap(); @@ -15313,7 +15264,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let local_quarantine_id = Uuid::now_v7(); @@ -15356,7 +15307,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let info = service @@ -15387,7 +15338,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); let local_quarantine_id = Uuid::now_v7(); @@ -15452,7 +15403,7 @@ pub(crate) mod service_tests { std::fs::create_dir_all(&staging_dir).unwrap(); std::fs::write(staging_dir.join("vmstate"), b"partial").unwrap(); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let records = service .recover_prelaunch_startup_states_in(cache_dir.path(), &Default::default()) .unwrap(); @@ -15516,7 +15467,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); // Park the lock so the wake and the suspend queue behind it in a // deterministic order: wake first, suspend second. @@ -15576,7 +15527,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .wake(internal_request(WakeRequest { @@ -15604,7 +15555,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.auto_lifecycle.record_wake_completed(&sandbox_id); tokio::time::sleep(std::time::Duration::from_millis(20)).await; @@ -15643,7 +15594,7 @@ pub(crate) mod service_tests { ) .unwrap() ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .suspend(internal_request(SuspendRequest { @@ -15745,7 +15696,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = access_idle_last_touched(&service, &sandbox_id); service.begin_draining_for_restart(); @@ -15771,7 +15722,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service.begin_draining_for_restart(); let error = match service @@ -15879,7 +15830,7 @@ pub(crate) mod service_tests { conn, SandboxManagerServiceOptions { route_ed25519_verification_bundle: Some(Arc::new(verification_bundle)), - ..Default::default() + ..SandboxManagerServiceOptions::for_test() }, ); (service, route_token) @@ -16127,7 +16078,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &id, state); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .set_auto_lifecycle_policy_exact( &id, @@ -16192,7 +16143,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, 1_234); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = access_idle_last_touched(&service, &sandbox_id); let before_access = last_access_at(&service, &sandbox_id); @@ -16234,7 +16185,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let error = service .wake(internal_request(WakeRequest { @@ -16308,7 +16259,7 @@ pub(crate) mod service_tests { // Arrange. let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let capture = SandboxEventCapture::default(); let subscriber = tracing_subscriber::registry().with(capture.clone()); let guard = tracing::subscriber::set_default(subscriber); @@ -16470,7 +16421,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -16501,7 +16452,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for _ in 0..5 { let second = now_secs(); let sandbox = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -16551,7 +16502,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (_host_tx, _client_stream) = open_presence_test_stream(None); yield_to_forward_task().await; @@ -16573,7 +16524,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let seeded_epoch = access_idle_epoch(&service, &sandbox_id); let (host_tx, _client_stream) = @@ -16611,7 +16562,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let _policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let seeded_epoch = access_idle_epoch(&service, &sandbox_id); let (tick_observer, mut ticks) = tokio::sync::mpsc::unbounded_channel(); @@ -16654,7 +16605,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (_host_tx, client_stream) = open_presence_test_stream(Some(test_stream_presence(&service, &sandbox_id, 60, 3600))); @@ -16683,7 +16634,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); // cap = 2 intervals: ticks at +60s and +120s land (the +120s tick // observes elapsed >= cap and disarms), later ticks must not fire. @@ -16720,7 +16671,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); { let db = service.db.lock().expect("mutex poisoned"); crate::db::update_state(&db, &sandbox_id, SandboxState::Suspended) @@ -16752,7 +16703,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let stale = seed_due_access_idle_sandbox_state(&service, &sandbox_id, &policy, 300); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id); @@ -16790,7 +16741,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); let policy = seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let stale = seed_due_access_idle_sandbox_state(&service, &sandbox_id, &policy, 300); let (_host_tx, _client_stream) = @@ -16814,7 +16765,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); seed_due_access_idle_sandbox(&conn, &sandbox_id, 120, 300); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let (host_tx, mut client_stream) = open_presence_test_stream(Some(test_stream_presence(&service, &sandbox_id, 60, 3600))); @@ -16875,7 +16826,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Started); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for round in 0..32 { // Clears the wake-completion record (and the lock maps) so each @@ -16951,7 +16902,7 @@ pub(crate) mod service_tests { rusqlite::params![stale_updated_at, sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .run_due_auto_lifecycle(&sandbox_id) @@ -17038,7 +16989,7 @@ pub(crate) mod service_tests { rusqlite::params![stale_updated_at, sandbox_id], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .run_due_auto_lifecycle(&sandbox_id) @@ -17071,7 +17022,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let wake_lock = service.auto_lifecycle.wake_lock(&sandbox_id); let wake_guard = wake_lock.lock_owned().await; @@ -17110,7 +17061,7 @@ pub(crate) mod service_tests { let mut runtime_state = initial_runtime_state(&policy, now_secs().saturating_sub(10)); recompute_runtime_state(&mut runtime_state, &policy); crate::db::update_auto_lifecycle_state(&conn, &sandbox_id, &runtime_state).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let result = tokio::time::timeout( std::time::Duration::from_secs(60), @@ -17176,7 +17127,7 @@ pub(crate) mod service_tests { // and the still-refused failed variants are pinned by the // `checkpoint_start_keeps_refusing_failed_row_*` tests. let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); for state in [SandboxState::Created, SandboxState::Stopped] { let sandbox_id = SandboxId::new().to_string(); @@ -17215,7 +17166,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let before = service.lookup_sandbox(&sandbox_id).unwrap(); let mut request = checkpoint_start_request(sandbox_id.clone(), "suspend", None); @@ -17244,7 +17195,7 @@ pub(crate) mod service_tests { let source_sandbox_id = SandboxId::new().to_string(); let artifact_snapshot_id = SnapshotId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let response = service .start_sandbox( @@ -17302,7 +17253,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let mut service = service_with_runtime_liveness(conn, true); + let mut service = service_with_runtime_probe(conn, true); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 0.001, 1.0), )); @@ -17333,7 +17284,7 @@ pub(crate) mod service_tests { async fn missing_checkpoint_restore_stages_before_activation_admission() { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 100.0, 100.0), )); @@ -17363,7 +17314,10 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = service_with_runtime_liveness(conn, true); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = service_with_runtime_probe(conn, true); service .start_sandbox( @@ -17418,7 +17372,7 @@ pub(crate) mod service_tests { ) .unwrap() .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let uuid8 = service.lookup_sandbox(&sandbox_id).unwrap().uuid8; let cleanup_guard = service .activation_cleanup_guard( @@ -17503,7 +17457,7 @@ pub(crate) mod service_tests { insert_test_sandbox_with_runtime(&conn, &sandbox_id_string, SandboxState::Pausing); let stopping_id = SandboxId::new(); insert_test_sandbox(&conn, &stopping_id.to_string(), SandboxState::Stopping); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets(); @@ -17582,7 +17536,7 @@ pub(crate) mod service_tests { &cleanup_failed_id.to_string(), SandboxState::CleanupFailed, ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut targets = service.recover_requested_cleanup_targets(); targets.sort_by_key(|target| target.sandbox_id); @@ -17625,7 +17579,7 @@ pub(crate) mod service_tests { rusqlite::params![Uuid::now_v7().to_string(), &sandbox_id_string], ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17642,7 +17596,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Deleting); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17680,7 +17634,7 @@ pub(crate) mod service_tests { ), }; crate::auto_lifecycle_delete_receipts::db::insert_pending_receipt(&conn, receipt).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let targets = service.recover_requested_cleanup_targets_for_ids(vec![sandbox_id]); @@ -17723,7 +17677,7 @@ pub(crate) mod service_tests { CleanupCompletion::MarkStopped, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let owner = CleanupLockGuard::new( Arc::clone(&service.cleanup_locks), sandbox_id_string.clone(), @@ -17762,7 +17716,7 @@ pub(crate) mod service_tests { CleanupCompletion::Quarantine, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let incumbent = match service.try_cleanup_fence(&sandbox_id_string) { CleanupLockTryResult::Acquired(fence) => fence, CleanupLockTryResult::Busy => panic!("test fence must be available"), @@ -17895,7 +17849,7 @@ pub(crate) mod service_tests { ) .unwrap() .unwrap(); - let mut service = SandboxManagerService::new(conn); + let mut service = SandboxManagerService::for_test(conn); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 1_000.0, 1.0), )); @@ -17985,7 +17939,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); mark_test_remote_restore_artifact_staging(&conn, &sandbox_id); - let mut service = service_with_runtime_liveness(conn, false); + let mut service = service_with_runtime_probe(conn, false); service.activation_admission_limiter = Arc::new(ActivationAdmissionLimiter::with_config( activation_admission_config(1, 0.001, 1.0), )); @@ -18051,7 +18005,7 @@ pub(crate) mod service_tests { .unwrap() .require_applied() .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18084,7 +18038,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Failed); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); for _ in 0..2 { let error = service .start_sandbox( @@ -18117,7 +18071,7 @@ pub(crate) mod service_tests { "user requested delete", ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18150,7 +18104,7 @@ pub(crate) mod service_tests { [&sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18184,7 +18138,10 @@ pub(crate) mod service_tests { rusqlite::params![LocalArtifactId::new().to_string(), &sandbox_id], ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18224,7 +18181,7 @@ pub(crate) mod service_tests { crate::db::CleanupCompletion::DeleteRow, ) .unwrap(); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let _ = service .start_sandbox( @@ -18247,11 +18204,12 @@ pub(crate) mod service_tests { ); } - // Paused, like the sibling checkpoint-retry cases: the `Unavailable` this - // asserts is produced by the vol-manager connect exhausting its retry - // budget, which against an absent socket is ~9.5 s of `tokio::time` - // backoff. The connects still run and still fail; only the backoff between - // them is virtual. + // Paused, like the sibling checkpoint-retry cases. The `Unavailable` this + // asserts comes from restore staging under the fixed cache root: as an + // ordinary user, as CI runs it, the staging directory's `create_dir_all` + // fails; as root, the volume-manager connect that follows exhausts its + // `tokio::time` retry budget against an absent socket, which pausing + // makes virtual. #[tokio::test(start_paused = true)] async fn checkpoint_retry_preserves_row_for_resumable_restore() { // Attempt 1 of a cross-node restore can be cancelled mid-stage (the @@ -18264,7 +18222,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Restoring); mark_test_remote_restore_artifact_staging(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18319,7 +18277,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_test_sandbox(&conn, &sandbox_id, SandboxState::Failed); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let error = service .start_sandbox( @@ -18429,7 +18387,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, false); + let service = service_with_runtime_probe(conn, false); let error = service .start_sandbox( @@ -18456,7 +18414,7 @@ pub(crate) mod service_tests { let conn = crate::db::test_conn(); let sandbox_id = SandboxId::new().to_string(); insert_started_test_sandbox(&conn, &sandbox_id); - let service = service_with_runtime_liveness(conn, true); + let service = service_with_runtime_probe(conn, true); let error = service .start_sandbox( @@ -18488,7 +18446,10 @@ pub(crate) mod service_tests { crate::db::CleanupCompletion::MarkSuspended, ) .unwrap(); - let service = SandboxManagerService::new(conn); + // The test service's node-manager refuses every dial, so the + // retirement's cleanup cannot start and the row keeps the intent it + // persisted. + let service = SandboxManagerService::for_test(conn); let _ = service .start_sandbox( @@ -18533,7 +18494,7 @@ pub(crate) mod service_tests { "restore activation was interrupted", ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); add_test_monitoring(&service, &sandbox_id).await; service.auto_lifecycle.record_wake_completed(&sandbox_id); let pending = service.lookup_sandbox(&sandbox_id).unwrap(); @@ -18603,7 +18564,7 @@ pub(crate) mod service_tests { .unwrap(); let before = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &key, &before).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); install_authoritative_route_epoch( &mut service.route_stream_epochs.lock().unwrap(), id, @@ -18717,7 +18678,7 @@ pub(crate) mod service_tests { .unwrap(); let old = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &id.to_string(), &old).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let protection = { let mut epochs = service.route_stream_epochs.lock().unwrap(); install_authoritative_route_epoch(&mut epochs, id, RouteStreamEpochIdentity(identity)); @@ -18783,7 +18744,7 @@ pub(crate) mod service_tests { .unwrap(); let due = initial_runtime_state(&policy, now_secs().saturating_sub(1000)); crate::db::update_auto_lifecycle_state(&conn, &key, &due).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); install_authoritative_route_epoch( &mut service.route_stream_epochs.lock().unwrap(), id, @@ -18848,7 +18809,7 @@ pub(crate) mod service_tests { let key = id.to_string(); let policy = access_idle_policy(300); insert_test_sandbox_with_policy(&conn, &key, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut owners = Vec::new(); for generation in [1, 2] { let identity = route_identity(id, generation, SandboxActivationAttemptId::new()); @@ -18910,7 +18871,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle_delete_receipts::RouteIdentityProvenance::Unfenced, ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let now = now_secs(); let mut request = touch_auto_lifecycle_access_request(&id.to_string(), now); let extend = @@ -19059,7 +19020,7 @@ pub(crate) mod service_tests { crate::auto_lifecycle::extend_access_deadline(&mut runtime, &policy, now_secs() + 900) .unwrap(); crate::db::update_auto_lifecycle_state(&conn, &id, &runtime).unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); service .set_auto_lifecycle_policy_exact( &id, @@ -19098,7 +19059,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let policy = access_idle_policy(30); insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; let response = service @@ -19137,7 +19098,7 @@ pub(crate) mod service_tests { let sandbox_id = SandboxId::new().to_string(); let policy = access_idle_policy(30); insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; service @@ -19186,7 +19147,7 @@ pub(crate) mod service_tests { &AutoLifecycleRuntimeState::default(), ) .unwrap(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let observed_at = now_secs() + 600; let response = service @@ -19222,7 +19183,7 @@ pub(crate) mod service_tests { .unwrap() .expect("sandbox row") .auto_lifecycle_state; - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .touch_auto_lifecycle(internal_request(touch_auto_lifecycle_access_request( @@ -19251,7 +19212,7 @@ pub(crate) mod service_tests { SandboxState::Started, &AutoLifecyclePolicy::disabled(), ); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let response = service .touch_auto_lifecycle(internal_request(touch_auto_lifecycle_access_request( @@ -19469,7 +19430,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); let outcome = service @@ -19520,7 +19481,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let outcome = service .request_guest_active_for(&sandbox_id, Duration::from_secs(10)) @@ -19552,7 +19513,7 @@ pub(crate) mod service_tests { }], }; insert_test_sandbox_with_policy(&conn, &sandbox_id, SandboxState::Started, &policy); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let before = now_secs(); assert!( diff --git a/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs index 4b4ba154bc..858086d167 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/cancelled_cleanup_rpc_tests.rs @@ -1,4 +1,4 @@ -use super::service_tests::{insert_test_sandbox, runtime_fields, service_for_teardown_test}; +use super::service_tests::{insert_test_sandbox, runtime_fields}; use super::*; use crate::service::activation_support::internal_request; use pplx_space_common_core::types::ExecutionGeneration; @@ -22,18 +22,20 @@ async fn cancelled_cleanup_rpcs_preserve_admitted_termination_strength() { let sandbox_id = SandboxId::new(); let id = sandbox_id.to_string(); insert_test_sandbox(&conn, &id, SandboxState::Started); - let service = service_for_teardown_test(conn); + let run_dir = tempfile::tempdir().unwrap(); + let service = SandboxManagerService::new_with_options( + conn, + SandboxManagerServiceOptions::for_test_in(run_dir.path()), + ); service .update_runtime_fields( sandbox_id, runtime_fields(ExecutionGeneration::new(1).unwrap(), 7), ) .unwrap(); - let run_dir = tempfile::tempdir().unwrap(); let sandbox_dir = run_dir.path().join(&id); std::fs::create_dir(&sandbox_dir).unwrap(); std::os::unix::fs::symlink(run_dir.path(), sandbox_dir.join("fc.pid")).unwrap(); - *service.cleanup_executor.run_dir.lock().unwrap() = Some(run_dir.path().to_owned()); let barrier = Arc::new(crate::service::cleanup_executor::TestAdmissionBarrier { entered: tokio::sync::Notify::new(), release: tokio::sync::Semaphore::new(0), diff --git a/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs index 676f757818..8d852885ce 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/local_backup_inventory_tests.rs @@ -68,7 +68,7 @@ async fn pending_local_backups_carry_a_concrete_artifact_kind_for_both_origins() let stopped = SandboxId::new().to_string(); insert_test_sandbox(&conn, &suspended, SandboxState::Suspended); insert_test_sandbox(&conn, &stopped, SandboxState::Stopped); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let suspend_artifact = LocalArtifactId::new(); let stop_artifact = LocalArtifactId::new(); pin_artifact( @@ -130,7 +130,7 @@ async fn a_degraded_sandbox_still_reports_its_pending_artifact() { let conn = crate::db::test_conn(); let degraded = SandboxId::new().to_string(); insert_test_sandbox(&conn, °raded, SandboxState::Suspended); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let degraded_artifact = LocalArtifactId::new(); pin_artifact( &service, @@ -188,7 +188,7 @@ async fn a_degraded_sandbox_still_reports_its_pending_artifact() { #[tokio::test] async fn pending_local_backups_are_not_readable_by_the_general_observer_set() { let conn = crate::db::test_conn(); - let service = SandboxManagerService::new(conn); + let service = SandboxManagerService::for_test(conn); let mut request = Request::new(crate::proto::ListPendingLocalBackupsRequest {}); request.extensions_mut().insert(PeerIdentity::new( users::SPACE_NODEAPI.uid, diff --git a/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs b/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs index 879bbdfc5d..0ba568d985 100644 --- a/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs +++ b/crates/libs/sbx/mgr/server/src/service/tests/node_manager_transport_tests.rs @@ -1,9 +1,16 @@ //! Lifecycle handlers driven against an in-process node-manager. //! -//! The service is built with every host dependency redirected: the -//! node-manager connector reaches a [`FakeNodeManager`], the host-control and -//! credential-manager sockets are test-owned paths nothing binds, and no -//! runtime is ever live. +//! The service is built from [`SandboxManagerServiceOptions::for_test`], with +//! every host dependency redirected: +//! +//! - the node-manager connector reaches a [`FakeNodeManager`]; +//! - every template lookup goes to a template-manager fake whose socket is +//! unbound, so a lookup fails to connect (`TemplateLookupError::Connect`, +//! which create reports as `Internal` "cannot connect to template-manager"); +//! - the credential-manager socket is a test-owned path nothing binds; +//! - the host-control socket, which sandbox-manager only passes through to +//! node-manager's `Launch`, names no live host service; +//! - no runtime is ever live. use super::service_tests::{insert_test_sandbox, runtime_fields}; use super::*; @@ -14,52 +21,20 @@ use pplx_space_node_mgr_client::proto::{ CleanupSandboxRequest, CleanupSandboxResponse, StopProcessRequest, StopProcessResponse, }; -/// A service whose node-manager clients all come from `node_manager` and whose -/// template lookups all go to `template_resolver`. -pub(crate) fn isolated_service( +/// A test service whose node-manager clients all come from `node_manager`. +pub(crate) fn service_with_node_manager( conn: rusqlite::Connection, node_manager: NodeManagerConnector, - template_resolver: Arc, - sockets: &Path, ) -> SandboxManagerService { SandboxManagerService::new_with_options( conn, SandboxManagerServiceOptions { - host_control_socket_path: sockets.join("absent-host-control.sock"), - credential_manager_socket_path: sockets.join("absent-credential-manager.sock"), - sandbox_runtime_live: Arc::new(|_| false), - shutdown_token: CancellationToken::new(), - task_tracker: TaskTracker::new(), - local_node_id: Some(NodeId::from_uuid( - uuid::Uuid::parse_str("019f8000-0000-7000-8000-0000000000fe").unwrap(), - )), - node_role: None, - node_managed_by: None, - route_ed25519_verification_bundle: None, node_manager, - template_resolver, + ..SandboxManagerServiceOptions::for_test() }, ) } -/// [`isolated_service`] whose template lookups are all refused. -pub(crate) fn service_with_node_manager( - conn: rusqlite::Connection, - node_manager: NodeManagerConnector, - sockets: &Path, -) -> SandboxManagerService { - isolated_service( - conn, - node_manager, - Arc::new( - crate::template_resolver::fake::FakeTemplateResolver::refusing(Status::unavailable( - "template-manager is outside this test", - )), - ), - sockets, - ) -} - #[tokio::test] async fn fatal_kill_signal_failure_still_schedules_admitted_cleanup() { let stops = Arc::new(Mutex::new(Vec::new())); @@ -84,8 +59,7 @@ async fn fatal_kill_signal_failure_still_schedules_admitted_cleanup() { let sandbox_id = SandboxId::new(); let sandbox_id_string = sandbox_id.to_string(); insert_test_sandbox(&conn, &sandbox_id_string, SandboxState::Started); - let sockets = tempfile::tempdir().unwrap(); - let service = service_with_node_manager(conn, node_manager.connector(), sockets.path()); + let service = service_with_node_manager(conn, node_manager.connector()); service .update_runtime_fields( sandbox_id, diff --git a/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs index da09a3ef8c..d6f4855a90 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_gc/tests.rs @@ -13,7 +13,7 @@ fn test_ownership() -> GcOwnership { coordinator: crate::auto_lifecycle::AutoLifecycleCoordinator::default(), drain: Arc::new(crate::service::drain::DrainState::default()), shutdown: CancellationToken::new(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), } } @@ -113,11 +113,10 @@ fn full_disk() -> crate::disk_usage::DiskUsage { } } -/// Paused, like the sibling reaper tests: a reaped candidate's staging -/// directory is removed through node-manager, and against an absent socket -/// that connect spends its whole ~9.5 s retry budget on `tokio::time` backoff. -/// Nothing here reads elapsed time — eligibility is decided from the `now_secs` -/// this test passes in — so the clock only has to not be real. +/// Paused, like the sibling reaper tests. A reaped candidate's staging +/// directory is handed to node-manager, which here is an absent socket dialed +/// once. Nothing here reads elapsed time — eligibility is decided from the +/// `now_secs` this test passes in — so the clock only has to not be real. #[tokio::test(start_paused = true)] async fn malformed_artifact_times_are_retained_without_blocking_valid_siblings() { let conn = open_test_db(); diff --git a/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs index 56b73d4fdf..56733c2b65 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_restore/tests.rs @@ -1013,7 +1013,7 @@ async fn admission_rejection_preserves_completed_restore_work_for_retry() { &stage, target, &error, - &crate::node_manager_transport::NodeManagerConnector::host(), + &crate::node_manager_transport::fake::unbound_in_absent_dir(), ) .await; diff --git a/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs index a20f7911d1..40e2ffa812 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_scheduler/tests.rs @@ -143,7 +143,7 @@ fn checkpoint_context(state: SandboxState) -> (CheckpointContext, String) { db: Arc::new(std::sync::Mutex::new(conn)), auto_lifecycle: crate::auto_lifecycle::AutoLifecycleCoordinator::default(), health_lifecycle: crate::watchdog::HealthLifecycleEpochs::default(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), }, sandbox_id, ) diff --git a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs index c164544270..fea0e9c866 100644 --- a/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs +++ b/crates/libs/sbx/mgr/server/src/snapshot_upload_jobs/tests.rs @@ -37,7 +37,7 @@ async fn upload_executor_converges_unstaged_metadata_failure() { let executor = SnapshotUploadExecutor { db: std::sync::Arc::new(std::sync::Mutex::new(open_backfill_test_db())), drain_state: crate::service::tests::closed_drain_for_test(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -185,7 +185,7 @@ async fn executor_rejects_non_snapshot_upload_job_kind() { rusqlite::Connection::open_in_memory().unwrap(), )), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -218,7 +218,7 @@ async fn executor_requires_begin_before_volume_staging() { rusqlite::Connection::open_in_memory().unwrap(), )), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, )), @@ -364,7 +364,7 @@ fn ready_backup_keeps_completed_evidence_when_bookkeeping_misses_execution_deadl executor: SnapshotUploadExecutor { db: std::sync::Arc::clone(&db), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new( crate::retained_source::FakeInspector::new( crate::retained_source::RetainedSourceCheck::Absent, @@ -456,7 +456,7 @@ async fn maintenance_retries_failed_pruning_on_the_next_tick() { let worker = build_worker( std::sync::Arc::clone(&db), std::sync::Arc::clone(&drain), - crate::node_manager_transport::NodeManagerConnector::host(), + crate::node_manager_transport::fake::unbound_in_absent_dir(), ); let shutdown = tokio_util::sync::CancellationToken::new(); let maintenance = tokio::spawn(run_maintenance( @@ -1137,7 +1137,7 @@ fn ready_backup_returns_before_the_retained_source_is_inspected() { let executor = SnapshotUploadExecutor { db: std::sync::Arc::clone(&db), drain_state: std::sync::Arc::new(crate::service::drain::DrainState::default()), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), retained_source_inspector: std::sync::Arc::new(GatedInspector { gate: std::sync::Arc::clone(&gate), }), diff --git a/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs b/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs index 1fed44511c..178ff87a94 100644 --- a/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs +++ b/crates/libs/sbx/mgr/server/src/template_resolver/fake.rs @@ -1,6 +1,7 @@ //! A scripted template resolver for tests. -use std::sync::{Mutex, PoisonError}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; use pplx_space_template_mgr_client::proto::{GetTemplateRequest, GetTemplateResponse}; @@ -106,3 +107,54 @@ impl TemplateResolver for FakeTemplateResolver { }) } } + +/// A resolver whose lookups all dial `socket`, a test-owned path nothing +/// binds. As when template-manager is down, a dialed lookup fails with a +/// connect error and a shared-channel lookup with `Unavailable`. The dial is +/// made once, without the host client's connect-retry budget. +pub(crate) fn unbound(socket: &Path) -> Arc { + Arc::new(Unbound(socket.to_owned())) +} + +struct Unbound(PathBuf); + +fn endpoint() -> tonic::transport::Endpoint { + tonic::transport::Endpoint::from_static("http://[::]:50051") +} + +fn client( + channel: tonic::transport::Channel, +) -> pplx_space_template_mgr_client::TemplateManagerGrpcClient { + pplx_space_template_mgr_client::proto::template_manager_client::TemplateManagerClient::new( + pplx_space_common_grpc::trace_grpc_client_transport(channel), + ) +} + +#[async_trait::async_trait] +impl TemplateResolver for Unbound { + async fn get_template( + &self, + request: tonic::Request, + ) -> Result, TemplateLookupError> { + let channel = pplx_space_common_grpc::connect_uds_channel_with_endpoint( + "template-manager", + &self.0, + endpoint(), + ) + .await + .map_err(pplx_space_template_mgr_client::ConnectError::from)?; + Ok(client(channel).get_template(request).await?) + } + + async fn get_template_on_shared_channel( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + client(pplx_space_common_grpc::connect_uds_channel_lazy( + &self.0, + endpoint(), + )) + .get_template(request) + .await + } +} diff --git a/crates/libs/sbx/mgr/server/src/watchdog.rs b/crates/libs/sbx/mgr/server/src/watchdog.rs index cc0ba174cf..e1a3ff38d5 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog.rs @@ -808,12 +808,14 @@ impl SandboxWatchdog { ) -> Self { let auto_lifecycle = AutoLifecycleCoordinator::default(); let lifecycle_epochs = HealthLifecycleEpochs::default(); + // The loops tick immediately; their node-manager is an absent socket. + let node_manager = crate::node_manager_transport::fake::unbound_in_absent_dir(); let snapshot_scheduler = Arc::new(SnapshotScheduler::new(Arc::new( crate::snapshot_scheduler::CheckpointContext { db: db.clone(), auto_lifecycle: auto_lifecycle.clone(), health_lifecycle: lifecycle_epochs.clone(), - node_manager: NodeManagerConnector::host(), + node_manager: node_manager.clone(), }, ))); Self::new( @@ -826,7 +828,7 @@ impl SandboxWatchdog { shutdown_token, Arc::new(crate::service::drain::DrainState::default()), task_tracker, - NodeManagerConnector::host(), + node_manager, ) } diff --git a/crates/libs/sbx/mgr/server/src/watchdog/tests.rs b/crates/libs/sbx/mgr/server/src/watchdog/tests.rs index 1f0f70b921..f54e51cbb3 100644 --- a/crates/libs/sbx/mgr/server/src/watchdog/tests.rs +++ b/crates/libs/sbx/mgr/server/src/watchdog/tests.rs @@ -1103,7 +1103,7 @@ async fn pause_failed_recovery_fixture( db: db.clone(), auto_lifecycle: auto_lifecycle.clone(), health_lifecycle: lifecycle_epochs.clone(), - node_manager: crate::node_manager_transport::NodeManagerConnector::host(), + node_manager: crate::node_manager_transport::fake::unbound_in_absent_dir(), }, ))); snapshot_scheduler