Prepare template lookup scope for rolling upgrades

crates/libs/common/svc/src/lib.rs

@@ -38,6 +38,7 @@ pub mod space_api_route;
 pub mod space_api_route_capability;
 pub mod space_api_route_ed25519;
 pub mod sql;
+pub mod template_lookup;
 pub use pplx_space_systemd_watchdog as systemd;
 pub use pplx_space_tokio_runtime as tokio_runtime;
 pub mod types;

crates/libs/common/svc/src/template_lookup.rs

@@ -0,0 +1,125 @@
+//! Template lookup scope propagation during a mixed-version rollout.
+//! Receipts confirm protocol support, not authorization. Existing access checks
+//! remain authoritative until a separate enforcement protocol is deployed.
+
+use std::net::IpAddr;
+
+use pplx_space_common_core::types::SandboxId;
+use tonic::{Request, Response, Status};
+
+const SCOPE_HEADER: &str = "x-space-template-lookup-scope";
+const RECEIPT_HEADER: &str = "x-space-template-lookup-scope-received";
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum TemplateLookupScope {
+    /// Online lookup using the service's catalog authority and optional org filter.
+    Cluster,
+    /// Original authenticated node before forwarding.
+    Node(IpAddr),
+    /// Sandbox identity and, when forwarded, its original node.
+    Sandbox {
+        id: SandboxId,
+        caller_node: Option<IpAddr>,
+    },
+    /// Execution requested by an authenticated node-local principal.
+    LocalExecution,
+}
+
+impl std::fmt::Display for TemplateLookupScope {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::Cluster => f.write_str("cluster"),
+            Self::Node(address) => write!(f, "node:{address}"),
+            Self::Sandbox { id, caller_node } => {
+                write!(f, "sandbox:{id}")?;
+                if let Some(address) = caller_node {
+                    write!(f, "@{address}")?;
+                }
+                Ok(())
+            }
+            Self::LocalExecution => f.write_str("local"),
+        }
+    }
+}
+
+impl std::str::FromStr for TemplateLookupScope {
+    type Err = Status;
+
+    fn from_str(value: &str) -> Result<Self, Self::Err> {
+        match value {
+            "cluster" => Ok(Self::Cluster),
+            "local" => Ok(Self::LocalExecution),
+            _ => {
+                if let Some(address) = value.strip_prefix("node:") {
+                    return address
+                        .parse()
+                        .map(Self::Node)
+                        .map_err(|_| Status::invalid_argument("invalid template caller address"));
+                }
+                if let Some(id) = value.strip_prefix("sandbox:") {
+                    let (id, address) = id
+                        .split_once('@')
+                        .map_or((id, None), |(id, address)| (id, Some(address)));
+                    return Ok(Self::Sandbox {
+                        id: id.parse().map_err(|_| {
+                            Status::invalid_argument("invalid template sandbox identity")
+                        })?,
+                        caller_node: address.map(str::parse).transpose().map_err(|_| {
+                            Status::invalid_argument("invalid template caller address")
+                        })?,
+                    });
+                }
+                Err(Status::invalid_argument("invalid template lookup scope"))
+            }
+        }
+    }
+}
+
+impl TemplateLookupScope {
+    pub fn from_request<T>(request: &Request<T>) -> Result<Option<Self>, Status> {
+        request
+            .metadata()
+            .get(SCOPE_HEADER)
+            .map(|value| {
+                value
+                    .to_str()
+                    .map_err(|_| Status::invalid_argument("invalid template lookup scope"))?
+                    .parse()
+            })
+            .transpose()
+    }
+
+    pub fn attach<T>(self, request: &mut Request<T>) {
+        request.metadata_mut().insert(
+            SCOPE_HEADER,
+            self.to_string().parse().expect("scope is ASCII"),
+        );
+    }
+
+    /// A receipt must not satisfy the authorization acknowledgement used by strict peers.
+    pub fn receipt<T>(self, response: &mut Response<T>) {
+        response.metadata_mut().insert(
+            RECEIPT_HEADER,
+            self.to_string().parse().expect("scope is ASCII"),
+        );
+    }
+
+    pub fn observe_response<T>(self, response: &Response<T>) -> Result<(), Status> {
+        match response.metadata().get(RECEIPT_HEADER) {
+            None => {
+                pplx_space_metrics::auth::template_lookup_compatibility("missing_receipt");
+                tracing::warn!(
+                    "template lookup peer has no scope receipt; retaining legacy behavior"
+                );
+                Ok(())
+            }
+            Some(value) if value.to_str().ok() == Some(self.to_string().as_str()) => {
+                pplx_space_metrics::auth::template_lookup_compatibility("received");
+                Ok(())
+            }
+            Some(_) => Err(Status::failed_precondition(
+                "template lookup scope receipt mismatch",
+            )),
+        }
+    }
+}

crates/libs/internal/api/server/src/adapter.rs

@@ -1,4 +1,5 @@
 use pplx_space_common_core::types::OrgId;
+use pplx_space_common_svc::template_lookup::TemplateLookupScope;
 
 use super::metadata::*;
 use super::status::*;
@@ -394,6 +395,10 @@
         request: Request<FindTemplateReleaseRequest>,
     ) -> Result<Response<FindTemplateReleaseResponse>, Status> {
         let scope = self.template_read_scope(&request).await?;
+        let delegation = TemplateLookupScope::from_request(&request)?;
+        if delegation.is_none() {
+            pplx_space_metrics::auth::template_lookup_compatibility("missing_scope");
+        }
         let request = request.into_inner();
         let template_name = trimmed_required(&request.template_name, "template_name")?;
         let runtime = validated_template_runtime(&request.runtime)?;
@@ -454,20 +459,28 @@
             &template_name,
         )
         .await;
-        Ok(Response::new(FindTemplateReleaseResponse {
+        let mut response = Response::new(FindTemplateReleaseResponse {
             generic_authorized,
             authorized_org_id: caller_org.map(|org| org.to_string()),
             release: Some(release),
             storage_locations,
             template_visibility: visibility.into(),
-        }))
+        });
+        if let Some(scope) = delegation {
+            scope.receipt(&mut response);
+        }
+        Ok(response)
     }
 
     async fn find_template_release_by_hash(
         &self,
         request: Request<FindTemplateReleaseByHashRequest>,
     ) -> Result<Response<FindTemplateReleaseResponse>, Status> {
         let scope = self.template_read_scope(&request).await?;
+        let delegation = TemplateLookupScope::from_request(&request)?;
+        if delegation.is_none() {
+            pplx_space_metrics::auth::template_lookup_compatibility("missing_scope");
+        }
         let request = request.into_inner();
         let template_name = trimmed_required(&request.template_name, "template_name")?;
         // Canonical `sha256:`-prefixed form.
@@ -538,13 +551,17 @@
             &template_name,
         )
         .await;
-        Ok(Response::new(FindTemplateReleaseResponse {
+        let mut response = Response::new(FindTemplateReleaseResponse {
             generic_authorized,
             authorized_org_id: caller_org.map(|org| org.to_string()),
             release: Some(release),
             storage_locations,
             template_visibility: visibility.into(),
-        }))
+        });
+        if let Some(scope) = delegation {
+            scope.receipt(&mut response);
+        }
+        Ok(response)
     }
 
     async fn list_latest_template_releases(

crates/libs/metrics/src/auth.rs

@@ -100,3 +100,12 @@ pub fn uds_peer_exe_deleted(service: &str, principal: &str) {
         &[tag("service", service), tag("principal", principal)],
     );
 }
+
+/// Per-hop scope protocol support; a receipt does not certify authorization.
+pub fn template_lookup_compatibility(result: &str) {
+    counter(
+        "auth.template_lookup.compatibility",
+        1,
+        &[tag("result", result)],
+    );
+}

crates/libs/node/api/server/src/builder.rs

@@ -152,9 +152,12 @@ impl BuilderApi for BuilderApiService {
 
     async fn get_template(
         &self,
-        r: Request<pplx_space_template_mgr_client::proto::GetTemplateRequest>,
+        mut r: Request<pplx_space_template_mgr_client::proto::GetTemplateRequest>,
     ) -> Result<Response<pplx_space_template_mgr_client::proto::GetTemplateResponse>, Status> {
-        fwd_retry!("template-manager", self.clients.tmpl(), get_template, r)
+        let scope = crate::template_lookup::bind_caller_scope(&mut r)?;
+        let response = fwd_retry!("template-manager", self.clients.tmpl(), get_template, r)?;
+        scope.observe_response(&response)?;
+        Ok(response)
     }
 
     async fn update_template_settings(
@@ -244,11 +247,12 @@ impl BuilderApi for BuilderApiService {
 
     async fn pull_template(
         &self,
-        r: Request<crate::proto::PullTemplateRequest>,
+        mut r: Request<crate::proto::PullTemplateRequest>,
     ) -> Result<Response<crate::proto::PullTemplateResponse>, Status> {
         pplx_space_common_svc::auth::template_transfer::require_template_transfer_authority(&r)?;
+        let scope = crate::template_lookup::bind_caller_scope(&mut r)?;
         let req = r.into_inner();
-        let resp = crate::transfer::pull_template(&mut self.clients.tmpl(), &req).await?;
+        let resp = crate::transfer::pull_template(&mut self.clients.tmpl(), &req, scope).await?;
         Ok(Response::new(resp))
     }
 

crates/libs/node/api/server/src/clients/policy_auth_tests.rs

@@ -118,6 +118,12 @@ pub(super) fn auth_layer(
 
 pub(super) fn bearer<T>(body: T, token: &str) -> Request<T> {
     let mut request = Request::new(body);
+    request
+        .extensions_mut()
+        .insert(tonic::transport::server::TcpConnectInfo {
+            local_addr: None,
+            remote_addr: Some(([127, 0, 0, 1], 1234).into()),
+        });
     request.metadata_mut().insert(
         "authorization",
         format!("Bearer {token}")
@@ -139,7 +145,9 @@ enum Rpc {
 
 #[tokio::test]
 async fn policy_rpc_authentication_reaches_handlers() {
-    let (clients, calls, shutdown, mut servers) = backend().await;
+    let forwarded_scope = Arc::new(std::sync::Mutex::new(None));
+    let (clients, calls, shutdown, mut servers) =
+        backend_with_scope(Some(forwarded_scope.clone())).await;
     for (k8s, username, token, trusted) in [
         (true, "space-api", PROJECTED_TOKEN, true),
         (true, "workers", PROJECTED_TOKEN, false),
@@ -261,6 +269,15 @@ async fn policy_rpc_authentication_reaches_handlers() {
                         .await
                         .map(|_| ()),
                 };
+                if matches!(rpc, Rpc::Create) && result.is_ok() {
+                    use pplx_space_common_svc::template_lookup::TemplateLookupScope;
+                    let expected = if trusted {
+                        TemplateLookupScope::Cluster
+                    } else {
+                        TemplateLookupScope::Node("127.0.0.1".parse().unwrap())
+                    };
+                    assert_eq!(*forwarded_scope.lock().unwrap(), Some(expected.to_string()));
+                }
                 let requires_authority = match rpc {
                     Rpc::Network | Rpc::Lifecycle => true,
                     Rpc::Create | Rpc::Adopt => network || lifecycle || generation != 0,
@@ -299,13 +316,25 @@ pub(super) async fn backend() -> (
     Arc<AtomicUsize>,
     tokio_util::sync::CancellationToken,
     tokio::task::JoinSet<()>,
+) {
+    backend_with_scope(None).await
+}
+
+async fn backend_with_scope(
+    forwarded_scope: Option<Arc<std::sync::Mutex<Option<String>>>>,
+) -> (
+    BackendClients,
+    Arc<AtomicUsize>,
+    tokio_util::sync::CancellationToken,
+    tokio::task::JoinSet<()>,
 ) {
     let calls = Arc::new(AtomicUsize::new(0));
     let recorded_calls = calls.clone();
     // Acknowledging backend: success and the count prove a real handler forwarded
     // the request; denial must leave it untouched. No manager/database is needed.
     let backend = Router::new().fallback(any(move |request: http::Request<Body>| {
         let calls = recorded_calls.clone();
+        let scope = forwarded_scope.clone();
         async move {
             if request.uri().path().ends_with("/ExportTemplate") {
                 assert_eq!(
@@ -316,8 +345,22 @@ pub(super) async fn backend() -> (
                     "granted"
                 );
             }
+            if let Some(scope) = scope {
+                *scope.lock().expect("scope recorder lock") = request
+                    .headers()
+                    .get("x-space-template-lookup-scope")
+                    .map(|value| value.to_str().expect("ASCII scope header").to_owned());
+            }
             calls.fetch_add(1, Ordering::SeqCst);
+            let lookup_scope = request
+                .headers()
+                .get("x-space-template-lookup-scope")
+                .cloned();
             let payload = if request.uri().path().ends_with("/GetTemplate") {
+                assert!(
+                    lookup_scope.is_some(),
+                    "cache probes must carry authenticated scope"
+                );
                 use prost::Message;
                 tmpl::GetTemplateResponse {
                     template: Some(tmpl::TemplateRef {
@@ -339,9 +382,13 @@ pub(super) async fn backend() -> (
                     .to_be_bytes(),
             );
             frame.extend(payload);
-            http::Response::builder()
+            let mut response = http::Response::builder()
                 .header("content-type", "application/grpc")
-                .header("grpc-status", "0")
+                .header("grpc-status", "0");
+            if let Some(scope) = lookup_scope {
+                response = response.header("x-space-template-lookup-scope-received", scope);
+            }
+            response
                 .body(Body::from(frame))
                 .expect("valid gRPC acknowledgement")
         }

crates/libs/node/api/server/src/clients/template_transfer_auth_tests.rs

@@ -98,6 +98,13 @@ async fn template_transfer_authority_survives_the_wire(
                 template_id: "example-template".into(),
                 content_hash: Some(format!("sha256:{}", "a".repeat(64))),
                 skip_if_exists: true,
+                sources: vec![crate::proto::TransferSource {
+                    source: Some(crate::proto::transfer_source::Source::Node(
+                        crate::proto::NodeSource {
+                            address: "http://127.0.0.1:1".into(),
+                        }
+                    )),
+                }],
                 ..Default::default()
             }
         ),

crates/libs/node/api/server/src/lib.rs

@@ -15,6 +15,7 @@ pub mod node_ctl;
 pub mod node_info;
 mod restart_handoff;
 mod run;
+mod template_lookup;
 pub mod transfer;
 pub mod uds_per_sandbox_limiter;
 pub mod worker;