Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions crates/libs/api/src/controllers/sandbox/lifecycle.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use pplx_space_common_core::types::SandboxId;

Check warning on line 1 in crates/libs/api/src/controllers/sandbox/lifecycle.rs

View workflow job for this annotation

GitHub Actions / rust_static / Rust clippy

[archcheck] reported by reviewdog 🐶 crates/libs/api/src/controllers/sandbox/lifecycle.rs shrank from 1815 to 1814 code lines. Lower its cap in architecture-baseline.toml to 1814. Raw Output: message:"crates/libs/api/src/controllers/sandbox/lifecycle.rs shrank from 1815 to 1814 code lines. Lower its cap in architecture-baseline.toml to 1814." location:{path:"crates/libs/api/src/controllers/sandbox/lifecycle.rs" range:{start:{line:1}}} severity:WARNING source:{name:"archcheck"} code:{value:"file_size.stale_entries"}

use super::*;
use crate::controllers::db_contention_error;
Expand Down Expand Up @@ -2208,14 +2208,15 @@
SandboxLifecycleStatus::Started
}
SandboxAction::Wake | SandboxAction::ImplicitWake(_) => {
let mut client = self
.substrate
// The worker serves Wake only on its internal surface. With no
// row there is no start context to send.
self.substrate
.runtime_client(
typed_id,
crate::controllers::substrate::RecoveryCharge::Exempt,
)
.await?;
client
.await?
.internal_client()
.wake_sandbox(WakeRequest {
implicit_wake_source: action
.wake_activity()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,11 @@ async fn start_sim_worker(pool: &PgPool) -> TestResult<(SimWorkerHarness, NodeId
.with_internal_api()
.start()
.await?;
register_direct_worker(pool, node_id, harness.endpoint()).await?;
Ok((harness, node_id))
}

async fn register_direct_worker(pool: &PgPool, node_id: NodeId, endpoint: &str) -> TestResult {
sqlx::query(sql!(
"INSERT INTO nodes (id, address, role, managed_by, state)
VALUES ($1, '127.0.0.1', 'worker', 'space', 'ready')"
Expand All @@ -190,10 +195,10 @@ async fn start_sim_worker(pool: &PgPool) -> TestResult<(SimWorkerHarness, NodeId
"INSERT INTO node_direct (node_id, node_api_endpoint) VALUES ($1, $2)"
))
.bind(node_id)
.bind(harness.endpoint())
.bind(endpoint)
.execute(pool)
.await?;
Ok((harness, node_id))
Ok(())
}

async fn insert_resident(state: &SimState, sandbox_id: SandboxId, status: &str) -> TestResult {
Expand Down Expand Up @@ -1300,6 +1305,129 @@ async fn untracked_actions_report_status_and_leave_the_row() -> TestResult {
Ok(())
}

/// Records every Wake and the bearer it arrived with. It serves only the
/// internal worker service, so a call that reached any other one fails.
#[derive(Clone, Default)]
struct RecordingInternalWorker {
wakes: std::sync::Arc<std::sync::Mutex<Vec<RecordedWake>>>,
}

type RecordedWake = (
WakeRequest,
Option<tonic::metadata::MetadataValue<tonic::metadata::Ascii>>,
);

#[tonic::async_trait]
impl pplx_space_node_api_client::proto::worker_api_internal_server::WorkerApiInternal
for RecordingInternalWorker
{
async fn adopt_pool_sandbox(
&self,
_request: tonic::Request<pplx_space_sbx_mgr_client::proto::AdoptPoolSandboxRequest>,
) -> Result<
tonic::Response<pplx_space_sbx_mgr_client::proto::AdoptPoolSandboxResponse>,
tonic::Status,
> {
Err(tonic::Status::unimplemented("not used by this test"))
}

async fn wake_sandbox(
&self,
request: tonic::Request<WakeRequest>,
) -> Result<tonic::Response<pplx_space_sbx_mgr_client::proto::WakeResponse>, tonic::Status>
{
let bearer = request.metadata().get("authorization").cloned();
let request = request.into_inner();
self.wakes.lock().unwrap().push((request.clone(), bearer));
Ok(tonic::Response::new(
pplx_space_sbx_mgr_client::proto::WakeResponse {
sandbox_id: request.sandbox_id,
transition: pplx_space_sbx_mgr_client::proto::WakeTransition::Resume as i32,
route_identity: None,
},
))
}
}

/// With no row to build a start context from, an untracked wake sends the
/// worker's internal Wake a bare request under the control-plane bearer,
/// the call a tracked wake makes with its context.
#[tokio::test(flavor = "current_thread")]
async fn untracked_wake_uses_the_internal_worker_wake() -> TestResult {
use pplx_space_common_svc::bearer_token::{SpaceBearerToken, space_bearer_metadata};
use pplx_space_node_api_client::proto::worker_api_internal_server::WorkerApiInternalServer;

if !crate::test_support::enter_sim_worker_process_with_owned_token_files() {
return Ok(());
}
let db = pg_test_db_with_max_connections(3).await?;
let pool = &db.pool;
let controller = direct_controller(pool);
let worker = RecordingInternalWorker::default();
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
let endpoint = listener.local_addr()?.to_string();
let shutdown = tokio_util::sync::CancellationToken::new();
let server = tokio::spawn(
tonic::transport::Server::builder()
.add_service(WorkerApiInternalServer::new(worker.clone()))
.serve_with_incoming_shutdown(
tonic::transport::server::TcpIncoming::from(listener),
shutdown.clone().cancelled_owned(),
),
);
let node_id = NodeId::new();
register_direct_worker(pool, node_id, &endpoint).await?;

let explicit = SandboxId::new();
let implicit = SandboxId::new();
let activity = ExecutionActivityKind::Command;
for (sandbox_id, action) in [
(explicit, SandboxAction::Wake),
(implicit, SandboxAction::ImplicitWake(activity)),
] {
insert_sandbox(pool, sandbox_id, "created", "space").await?;
place_on_node(pool, sandbox_id, node_id).await?;
let body = Box::pin(run_action(
&controller,
pool,
sandbox_id,
None,
action,
None,
))
.await?;
assert_eq!(body["status"], "started", "{action:?}");
}
shutdown.cancel();
server.await??;

let control_bearer = space_bearer_metadata(SpaceBearerToken::NodeApiInternal)?;
let wakes = worker.wakes.lock().unwrap().clone();
assert_eq!(
wakes,
[
(
WakeRequest {
sandbox_id: explicit.to_string(),
blocking: true,
..Default::default()
},
Some(control_bearer.clone()),
),
(
WakeRequest {
sandbox_id: implicit.to_string(),
blocking: true,
implicit_wake_source: activity.wake_source(),
..Default::default()
},
Some(control_bearer),
),
]
);
Ok(())
}

// ---------------------------------------------------------------------------
// `delete` of an observed-only sandbox (`delete_observed_sandbox`).
// ---------------------------------------------------------------------------
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ impl RuntimeClient {
.await
}

pub(super) fn internal_client(&self) -> RuntimeInternalClient {
pub(in crate::controllers) fn internal_client(&self) -> RuntimeInternalClient {
WorkerApiInternalClient::new(node_api_channel(
self.channel.clone(),
SpaceBearerToken::NodeApiInternal,
Expand Down
15 changes: 8 additions & 7 deletions crates/libs/common/svc/src/auth/k8s.rs
Original file line number Diff line number Diff line change
Expand Up @@ -611,9 +611,10 @@ impl K8sTokenAuthConfig {
if role == NodeRole::Worker && self.node_api_trusted_wait_ready_usernames.is_empty() {
return Err(AuthConfigError::TrustedWaitReadyServiceAccountsMissing);
}
// Worker and builder both serve the policy-mutation RPCs, so both must
// ship a non-empty trusted policy allowlist. Fail closed at startup
// rather than silently admitting every allow-listed caller.
// A worker serves the policy-mutation RPCs, and both roles gate template
// deletion on this allowlist, so both must ship a non-empty one. Fail
// closed at startup rather than silently admitting every allow-listed
// caller.
if matches!(role, NodeRole::Worker | NodeRole::Builder)
&& self.node_api_trusted_policy_usernames.is_empty()
{
Expand Down Expand Up @@ -656,8 +657,9 @@ impl K8sTokenAuthConfig {

// Policy-mutation trust is an explicit, independently-configured
// allowlist (validated non-empty at startup for worker/builder). It is
// `Some` only on the surfaces that serve the policy RPCs; every other
// surface reports `None`, which the authority check treats as "deny".
// `Some` only on the surfaces that serve policy or template-mutation
// RPCs; every other surface reports `None`, which the authority check
// treats as "deny".
let trusted_policy_usernames = match (role, surface) {
// Internal pool adoption also accepts caller-supplied policy. Its
// caller must pass the internal allowlist as well as policy trust.
Expand Down Expand Up @@ -747,8 +749,7 @@ fn guest_bridge_method(role: NodeRole, path: &str) -> bool {
};
matches!(
method,
"SpacedPingGuest"
| "SpacedGetStatus"
"SpacedGetStatus"
| "SpacedWaitReady"
| "SpacedStat"
| "SpacedList"
Expand Down
6 changes: 3 additions & 3 deletions crates/libs/common/svc/src/auth/k8s/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ async fn data_plane_identity_cannot_reuse_guest_authority_for_control_or_transfe
("/space.node.api.v1.WorkerApi/ServicePortIngress", true),
("/space.node.api.v1.NodeInfo/GetNodeInfo", true),
("/space.node.api.v1.WorkerApi/CreateSandboxSnapshot", false),
("/space.node.api.v1.WorkerApi/ListSandboxes", false),
("/space.node.api.v1.WorkerApi/KillSandbox", false),
(
"/space.node.api.v1.WorkerApi/SpacedFuturePrivilegedMethod",
false,
Expand Down Expand Up @@ -311,7 +311,7 @@ async fn peer_node_identity_is_confined_to_template_export() {
reviewer_returning_status(config, token_review_status("space-node-api", username));
let token = projected_token();
for (method, allowed) in [
("/space.node.api.v1.WorkerApi/ListSandboxes", false),
("/space.node.api.v1.WorkerApi/KillSandbox", false),
("/space.node.api.v1.WorkerApi/NodeExportTemplate", true),
("/space.node.api.v1.WorkerApi/SandboxState", false),
("/space.node.api.v1.WorkerApi/SandboxStats", false),
Expand Down Expand Up @@ -351,7 +351,7 @@ async fn peer_node_identity_is_confined_to_template_export() {
.unwrap();
for (method, allowed) in [
("/space.node.api.v1.BuilderApi/NodeExportTemplate", true),
("/space.node.api.v1.BuilderApi/SandboxState", false),
("/space.node.api.v1.BuilderApi/DeleteTemplate", false),
("/space.node.api.v1.BuilderApi/BuildTemplate", false),
("/space.node.api.v1.WorkerApi/NodeExportTemplate", false),
] {
Expand Down
2 changes: 1 addition & 1 deletion crates/libs/common/svc/src/auth/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -260,7 +260,7 @@ pub enum AuthConfigError {
)]
TrustedWaitReadyServiceAccountsMissing,
#[error(
"Kubernetes token auth is enabled for a policy-serving node (worker or builder) but no trusted policy service accounts are configured"
"Kubernetes token auth is enabled for a worker or builder node-api but no trusted policy service accounts are configured"
)]
TrustedPolicyServiceAccountsMissing,
#[error("failed to initialize Kubernetes TokenReview client: {0}")]
Expand Down
Loading
Loading