sbx-mgr: inject the template-manager resolver

crates/libs/sbx/mgr/server/src/lib.rs

@@ -35,6 +35,7 @@ pub mod snapshot_scheduler;
 pub mod snapshot_upload;
 pub mod snapshot_upload_jobs;
 pub mod state;
+pub mod template_resolver;
 mod transfer_directory;
 pub mod watchdog;
 

crates/libs/sbx/mgr/server/src/service/auth_lookup.rs

@@ -1005,13 +1005,15 @@ impl SandboxManagerService {
         }
 
         if inherit_template_default {
-            let resolved_template =
-                resolve_template_paths(pplx_space_template_mgr_client::proto::TemplateRef {
+            let resolved_template = resolve_template_paths(
+                self.template_resolver.as_ref(),
+                pplx_space_template_mgr_client::proto::TemplateRef {
                     name: sandbox.template.clone(),
                     content_hash: sandbox.template_hash.clone(),
                     tag: None,
-                })
-                .await?;
+                },
+            )
+            .await?;
             policy = resolved_template.default_auto_lifecycle;
         }
 

crates/libs/sbx/mgr/server/src/service/create.rs

@@ -628,7 +628,7 @@ impl SandboxManagerService {
             "restore.template_prepare_source",
             START_TEMPLATE_RESOLUTION_TIMEOUT,
             id,
-            template_source_for_sandbox_prepare(sandbox),
+            template_source_for_sandbox_prepare(self.template_resolver.as_ref(), sandbox),
         )
         .await?;
         let prepare_template = prepare_source.template;

crates/libs/sbx/mgr/server/src/service/create_support.rs

@@ -564,21 +564,29 @@ pub(super) fn checkpoint_start_response_for_existing_started_sandbox(
 }
 
 pub(super) async fn resolve_template_paths(
+    templates: &dyn crate::template_resolver::TemplateResolver,
     template_ref: pplx_space_template_mgr_client::proto::TemplateRef,
 ) -> Result<ResolvedTemplatePaths, tonic::Status> {
-    let mut tmpl_client = pplx_space_template_mgr_client::connect()
+    use crate::template_resolver::TemplateLookupError;
+    let inner = templates
+        .get_template(tonic::Request::new(
+            pplx_space_template_mgr_client::proto::GetTemplateRequest {
+                caller_org_id: None,
+                template: Some(template_ref),
+                vcpu: None,
+                memory_mib: None,
+            },
+        ))
         .await
-        .map_err(|e| tonic::Status::internal(format!("cannot connect to template-manager: {e}")))?;
-    let resp = tmpl_client
-        .get_template(pplx_space_template_mgr_client::proto::GetTemplateRequest {
-            caller_org_id: None,
-            template: Some(template_ref),
-            vcpu: None,
-            memory_mib: None,
-        })
-        .await
-        .map_err(|e| tonic::Status::new(e.code(), format!("GetTemplate failed: {e}")))?;
-    let inner = resp.into_inner();
+        .map_err(|error| match error {
+            TemplateLookupError::Connect(e) => {
+                tonic::Status::internal(format!("cannot connect to template-manager: {e}"))
+            }
+            TemplateLookupError::Rpc(e) => {
+                tonic::Status::new(e.code(), format!("GetTemplate failed: {e}"))
+            }
+        })?
+        .into_inner();
     let tmpl = inner.template.unwrap_or_default();
     let arch = template_metadata_arch(&inner.template_dir).unwrap_or(inner.arch);
     let spaced_compatibility = template_metadata_spaced_compatibility(&inner.template_dir);
@@ -670,6 +678,7 @@ pub(super) fn stage_base_snapshot_runtime_metadata(
 }
 
 pub(super) async fn resolve_template_paths_for_start(
+    templates: &dyn crate::template_resolver::TemplateResolver,
     template_ref: pplx_space_template_mgr_client::proto::TemplateRef,
 ) -> Result<ResolvedTemplatePaths, Status> {
     // Start restores an existing sandbox's identity. A name-only lookup could
@@ -683,7 +692,7 @@ pub(super) async fn resolve_template_paths_for_start(
             "cannot recover sandbox artifact without its saved template hash",
         ));
     }
-    resolve_template_paths(template_ref)
+    resolve_template_paths(templates, template_ref)
         .await
         .map_err(start_template_resolution_failed_status)
 }
@@ -701,18 +710,22 @@ fn start_template_resolution_failed_status(error: Status) -> Status {
 }
 
 pub(super) async fn template_runtime_defaults_for_sandbox_start(
+    templates: &dyn crate::template_resolver::TemplateResolver,
     sandbox: &crate::db::SandboxRow,
     prepared_template_dir: Option<&str>,
 ) -> TemplateRuntimeDefaults {
     if let Some(template_dir) = prepared_template_dir {
         return template_runtime_defaults(template_dir);
     }
 
-    match resolve_template_paths(pplx_space_template_mgr_client::proto::TemplateRef {
-        name: sandbox.template.clone(),
-        content_hash: sandbox.template_hash.clone(),
-        tag: None,
-    })
+    match resolve_template_paths(
+        templates,
+        pplx_space_template_mgr_client::proto::TemplateRef {
+            name: sandbox.template.clone(),
+            content_hash: sandbox.template_hash.clone(),
+            tag: None,
+        },
+    )
     .await
     {
         Ok(resolved) => resolved.runtime_defaults,
@@ -742,6 +755,7 @@ pub(super) fn sandbox_rootfs_exists_in(cache_dir: &std::path::Path, sandbox_id:
 }
 
 pub(super) async fn template_source_for_sandbox_prepare(
+    templates: &dyn crate::template_resolver::TemplateResolver,
     sandbox: &crate::db::SandboxRow,
 ) -> Result<SandboxPrepareSource, Status> {
     if sandbox_rootfs_exists(&sandbox.id) {
@@ -764,13 +778,15 @@ pub(super) async fn template_source_for_sandbox_prepare(
         )));
     }
 
-    let resolved_template =
-        resolve_template_paths_for_start(pplx_space_template_mgr_client::proto::TemplateRef {
+    let resolved_template = resolve_template_paths_for_start(
+        templates,
+        pplx_space_template_mgr_client::proto::TemplateRef {
             name: sandbox.template.clone(),
             content_hash: sandbox.template_hash.clone(),
             tag: None,
-        })
-        .await?;
+        },
+    )
+    .await?;
     reject_incompatible_template_arch(
         &sandbox.template,
         resolved_template

crates/libs/sbx/mgr/server/src/service/create_support/tests.rs

@@ -476,17 +476,23 @@ fn restore_provenance_classifies_legacy_remote_metadata() {
 
 #[tokio::test]
 async fn existing_sandbox_cannot_resolve_a_missing_artifact_by_latest_alias() {
-    // With no template-manager available, this must reject locally rather than
-    // asking it to pick the newest build after the sandbox lost its identity.
-    let error =
-        resolve_template_paths_for_start(pplx_space_template_mgr_client::proto::TemplateRef {
+    // This must reject locally rather than asking template-manager to pick
+    // the newest build after the sandbox lost its identity.
+    let templates = crate::template_resolver::fake::FakeTemplateResolver::refusing(
+        tonic::Status::not_found("a latest-alias lookup must not be sent"),
+    );
+    let error = resolve_template_paths_for_start(
+        &templates,
+        pplx_space_template_mgr_client::proto::TemplateRef {
             name: "app-runtime".into(),
             content_hash: None,
             tag: None,
-        })
-        .await
-        .err()
-        .expect("missing identity cannot select a replacement artifact");
+        },
+    )
+    .await
+    .err()
+    .expect("missing identity cannot select a replacement artifact");
     assert_eq!(error.code(), tonic::Code::FailedPrecondition);
     assert!(error.message().contains("saved template hash"));
+    assert!(templates.requests().is_empty());
 }

crates/libs/sbx/mgr/server/src/service/grpc_core.rs

@@ -197,13 +197,15 @@ impl SandboxManagerService {
 
             // Resolve template to content-addressed paths via template-manager.
             // If caller pinned a content_hash, use it; otherwise resolve newest.
-            let resolved_template =
-                resolve_template_paths(pplx_space_template_mgr_client::proto::TemplateRef {
+            let resolved_template = resolve_template_paths(
+                self.template_resolver.as_ref(),
+                pplx_space_template_mgr_client::proto::TemplateRef {
                     name: template.to_string(),
                     content_hash: req.content_hash.clone(),
                     tag: None,
-                })
-                .await?;
+                },
+            )
+            .await?;
             if let Some(policy) = template_network_policy_from_proto(
                 req.template_network_policy.as_ref(),
                 req.template_network_policy_generation,
@@ -1176,5 +1178,7 @@ fn project_template_catalog(
         .collect()
 }
 
+#[cfg(test)]
+mod create_template_lookup_tests;
 #[cfg(test)]
 mod tests;

crates/libs/sbx/mgr/server/src/service/grpc_core/create_template_lookup_tests.rs

@@ -0,0 +1,178 @@
+//! Create's auto-lifecycle authority checks, which run just before template
+//! lookup, against a scripted template resolver. Template-manager and
+//! node-manager are fakes, so a request that gets past a check stops at the
+//! resolver's scripted answer.
+
+use super::*;
+use crate::node_manager_transport::fake::FakeNodeManager;
+use crate::template_resolver::TemplateLookupError;
+use crate::template_resolver::fake::{FakeTemplateResolver, LookupChannel};
+use pplx_space_common_core::host::linux::identities::users;
+use pplx_space_common_svc::PeerIdentity;
+use pplx_space_net_common::proto::PolicyAuthority;
+use pplx_space_template_mgr_client::proto::{GetTemplateRequest, TemplateRef};
+
+/// The resolver's refusal, distinct from every refusal a check produces before
+/// lookup.
+const LOOKUP_REFUSAL: &str = "scripted template lookup refusal";
+
+fn refusing_lookup() -> FakeTemplateResolver {
+    FakeTemplateResolver::refusing(Status::not_found(LOOKUP_REFUSAL))
+}
+
+fn unreachable_template_manager() -> FakeTemplateResolver {
+    FakeTemplateResolver::new(|| {
+        let invalid = "http://[".parse::<http::Uri>().unwrap_err();
+        Err(TemplateLookupError::Connect(
+            pplx_space_template_mgr_client::ConnectError::InvalidUri(invalid),
+        ))
+    })
+}
+
+#[tokio::test]
+async fn create_checks_auto_lifecycle_authority_before_template_lookup() {
+    if !crate::service::tests::enter_route_test_process(&[
+        ("EPPO_SDK_KEY", None),
+        ("SPACE_FLAGD_ENABLED", None),
+    ]) {
+        return;
+    }
+    let forwarded = PeerIdentity::new(
+        users::SPACE_NODEAPI.uid,
+        users::SPACE_NODEAPI.primary_group.gid,
+        None,
+    );
+    let local = PeerIdentity::new(0, 0, None);
+    struct Case {
+        name: &'static str,
+        peer: PeerIdentity,
+        authority: i32,
+        templates: fn() -> FakeTemplateResolver,
+        code: tonic::Code,
+        message_prefix: &'static str,
+        message_contains: &'static str,
+        looks_up_template: bool,
+    }
+    let cases = [
+        Case {
+            name: "local authority from a forwarded peer",
+            peer: forwarded.clone(),
+            authority: PolicyAuthority::Local as i32,
+            templates: refusing_lookup,
+            code: tonic::Code::PermissionDenied,
+            message_prefix: "authority=local requires a local operator peer",
+            message_contains: "",
+            looks_up_template: false,
+        },
+        Case {
+            name: "unspecified authority",
+            peer: forwarded.clone(),
+            authority: PolicyAuthority::Unspecified as i32,
+            templates: refusing_lookup,
+            code: tonic::Code::InvalidArgument,
+            message_prefix: "policy authority proto value is unspecified",
+            message_contains: "",
+            looks_up_template: false,
+        },
+        Case {
+            name: "unknown authority",
+            peer: forwarded.clone(),
+            authority: 99,
+            templates: refusing_lookup,
+            code: tonic::Code::InvalidArgument,
+            message_prefix: "policy authority proto value 99 is unknown",
+            message_contains: "",
+            looks_up_template: false,
+        },
+        Case {
+            name: "cluster authority from a forwarded peer",
+            peer: forwarded.clone(),
+            authority: PolicyAuthority::Cluster as i32,
+            templates: refusing_lookup,
+            code: tonic::Code::NotFound,
+            message_prefix: "GetTemplate failed: ",
+            message_contains: LOOKUP_REFUSAL,
+            looks_up_template: true,
+        },
+        Case {
+            name: "local authority from a local peer",
+            peer: local,
+            authority: PolicyAuthority::Local as i32,
+            templates: refusing_lookup,
+            code: tonic::Code::NotFound,
+            message_prefix: "GetTemplate failed: ",
+            message_contains: LOOKUP_REFUSAL,
+            looks_up_template: true,
+        },
+        Case {
+            name: "accepted authority with template-manager unreachable",
+            peer: forwarded,
+            authority: PolicyAuthority::Cluster as i32,
+            templates: unreachable_template_manager,
+            code: tonic::Code::Internal,
+            message_prefix: "cannot connect to template-manager: ",
+            message_contains: "invalid gRPC endpoint URI",
+            looks_up_template: true,
+        },
+    ];
+    for case in cases {
+        let templates = Arc::new((case.templates)());
+        let node_manager = FakeNodeManager::default().serve();
+        let sockets = tempfile::tempdir().unwrap();
+        let service = crate::service::tests::node_manager_transport_tests::isolated_service(
+            crate::db::test_conn(),
+            node_manager.connector(),
+            Arc::clone(&templates) as Arc<dyn crate::template_resolver::TemplateResolver>,
+            sockets.path(),
+        );
+        let sandbox_id = SandboxId::new().to_string();
+        let mut request = Request::new(CreateRequest {
+            template: "seed".into(),
+            owner: "root@local".into(),
+            runtime: pplx_space_common_core::types::RUNTIME_FIRECRACKER_ALIAS.into(),
+            boot_mode: BootMode::Cold as i32,
+            sandbox_id: Some(sandbox_id.clone()),
+            auto_lifecycle: Some(Default::default()),
+            auto_lifecycle_authority: case.authority,
+            ..Default::default()
+        });
+        request.extensions_mut().insert(case.peer);
+
+        let error = service.create(request).await.expect_err(case.name);
+
+        assert_eq!(error.code(), case.code, "{}: {error}", case.name);
+        assert!(
+            error.message().starts_with(case.message_prefix)
+                && error.message().contains(case.message_contains),
+            "{}: {error}",
+            case.name
+        );
+        let lookups = templates.lookups();
+        let expected_lookups = if case.looks_up_template {
+            vec![GetTemplateRequest {
+                caller_org_id: None,
+                template: Some(TemplateRef {
+                    name: "seed".into(),
+                    content_hash: None,
+                    tag: None,
+                }),
+                vcpu: None,
+                memory_mib: None,
+            }]
+        } else {
+            Vec::new()
+        };
+        assert_eq!(templates.requests(), expected_lookups, "{}", case.name);
+        // Create dials template-manager for its lookup and sends no metadata.
+        for lookup in &lookups {
+            assert_eq!(lookup.channel, LookupChannel::Dialed, "{}", case.name);
+            assert!(lookup.metadata.is_empty(), "{}", case.name);
+        }
+        assert!(node_manager.calls().is_empty(), "{}", case.name);
+        assert!(
+            !crate::db::sandbox_exists(&service.db_conn().unwrap(), &sandbox_id).unwrap(),
+            "{}",
+            case.name
+        );
+    }
+}